{"run_id": "eval-01/readout", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 44.0, "target_rank": 279.5, "source_logit": 6.0, "target_logit": 4.15625, "top_ids": [38352, 8330, 3001, 3693, 7513, 1111, 6364, 45418, 11413, 15973, 19312, 10769], "top_tokens": ["uenta", " Australia", "ica", " American", " European", "ule", " English", ".currency", " currency", "currency", "urname", " Japanese"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 19.0, "target_rank": 213.0, "source_logit": 7.96875, "target_logit": 4.8125, "top_ids": [11413, 15973, 96694, 3001, 53661, 4787, 27908, 28130, 5088, 28453, 3693, 34710], "top_tokens": [" currency", "currency", "onis", "ica", "cla", " North", "ingo", "ija", "urrency", " Currency", " American", "_currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 2.0, "target_rank": 4582.5, "source_logit": 25.75, "target_logit": 8.625, "top_ids": [11413, 6323, 3263, 35602, 8330, 110839, 10769, 10465, 24357, 12463, 36911, 85466], "top_tokens": [" currency", " Japan", "\".", " currencies", " Australia", "\u5728\u65e5\u672c", " Japanese", ".\",", " overseas", " colour", ".\".", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 1.5, "target_rank": 420.0, "source_logit": 22.5, "target_logit": 13.25, "top_ids": [11413, 6323, 7214, 65001, 13224, 47412, 10769, 37198, 35602, 7855, 104109, 74508], "top_tokens": [" currency", " Japan", " foreign", "\"\u2014", " Britain", " foreigners", " Japanese", " Commonwealth", " currencies", " British", "\u53f0\u6e7e", "\u00bb."]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 1621.0, "target_rank": 969.0, "source_logit": 1.75, "target_logit": 2.328125, "top_ids": [55964, 21251, 603, 12751, 3776, 56537, 29921, 5737, 3731, 7513, 67476, 28453], "top_tokens": ["\u2011", "onia", "one", " Roman", " One", " _______,", "\u2010", " Group", " Red", " European", "'),('", " Currency"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 33.5, "target_rank": 137.0, "source_logit": 6.1875, "target_logit": 4.84375, "top_ids": [364, 7179, 11413, 18174, 479, 21251, 11766, 29991, 362, 2932, 105469, 32], "top_tokens": [" '", " International", " currency", "oret", " G", "onia", " Standard", "\u540d\u79f0", " A", " She", "\u79cd\u7c7b", "A"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 111.5, "target_rank": 2224.0, "source_logit": 16.0, "target_logit": 12.0, "top_ids": [11413, 35602, 13867, 5313, 3146, 3283, 30176, 10806, 5837, 6290, 29970, 12922], "top_tokens": [" currency", " currencies", " county", " national", " country", " city", " counties", " retail", " countries", " town", " tourist", " island"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 3.0, "target_rank": 248.0, "source_logit": 19.5, "target_logit": 14.375, "top_ids": [37198, 11413, 6323, 7855, 7513, 5837, 35602, 6489, 7214, 94936, 86049, 17672], "top_tokens": [" Commonwealth", " currency", " Japan", " British", " European", " countries", " currencies", " international", " foreign", " denomination", " monarchy", " euro"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 282.0, "target_rank": 1327.5, "source_logit": 4.0625, "target_logit": 1.9765625, "top_ids": [2598, 3693, 4229, 89478, 279, 1083, 15503, 4787, 5434, 4102, 3881, 46427], "top_tokens": [" called", " American", " Ge", "\u2026\n", " the", " also", " primarily", " North", "\u2026\n\n", "\u00a0", " known", "/is"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 100.0, "target_rank": 518.5, "source_logit": 5.53125, "target_logit": 3.40625, "top_ids": [1852, 5938, 271, 3693, 4787, 4102, 18174, 479, 4882, 362, 5055, 3639], "top_tokens": [" same", " German", "\n\n", " American", " North", "\u00a0", "oret", " G", " South", " A", " National", " United"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 517.0, "target_rank": 2637.0, "source_logit": 13.5, "target_logit": 10.375, "top_ids": [11413, 104142, 103408, 35602, 113883, 32488, 28453, 85466, 115160, 5313, 29991, 57048], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " currencies", "\u672c\u56fd", " euros", " Currency", " Euros", "\u5176\u4ed6\u56fd\u5bb6", " national", "\u540d\u79f0", "__;\n"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 165.5, "target_rank": 4786.0, "source_logit": 9.3125, "target_logit": 3.0, "top_ids": [104142, 116866, 103408, 113883, 112397, 99599, 106127, 101406, 109442, 106333, 116588, 110065], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u6b27\u5143", "\u672c\u56fd", "\u56fd\u5bb6\u6807\u51c6", "\u56fd\u5bb6", "\u6b27\u76df", "\u8d27\u5e01", "\u8d22\u653f\u90e8", "\u5916\u6c47", "\u6210\u5458\u56fd", "\u6c47\u7387"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 3.0, "target_rank": 12497.5, "source_logit": 22.25, "target_logit": -0.2412109375, "top_ids": [10769, 11413, 6323, 2598, 57340, 11192, 104142, 28453, 17692, 51466, 19551, 80189], "top_tokens": [" Japanese", " currency", " Japan", " called", " yen", " dollars", "\u4eba\u6c11\u5e01", " Currency", " dollar", "Japanese", " USD", " Dollars"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 20483.5, "source_logit": 23.625, "target_logit": -1.6953125, "top_ids": [10769, 6323, 57340, 11413, 51466, 101059, 104142, 17692, 26194, 19551, 28453, 11192], "top_tokens": [" Japanese", " Japan", " yen", " currency", "Japanese", "\u65e5\u672c", "\u4eba\u6c11\u5e01", " dollar", " Tokyo", " USD", " Currency", " dollars"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 9.0, "target_rank": 3860.0, "source_logit": 33.5, "target_logit": 8.1875, "top_ids": [11413, 104142, 11192, 35602, 28453, 17692, 10769, 57340, 6323, 80189, 101406, 112455], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " currencies", " Currency", " dollar", " Japanese", " yen", " Japan", " Dollars", "\u8d27\u5e01", "\u82f1\u9551"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 4.5, "target_rank": 20662.5, "source_logit": 27.625, "target_logit": -0.3046875, "top_ids": [104142, 10769, 57340, 101059, 6323, 11413, 101406, 35602, 81022, 51466, 110839, 28453], "top_tokens": ["\u4eba\u6c11\u5e01", " Japanese", " yen", "\u65e5\u672c", " Japan", " currency", "\u8d27\u5e01", " currencies", " Kyoto", "Japanese", "\u5728\u65e5\u672c", " Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 14.0, "target_rank": 9621.5, "source_logit": 20.25, "target_logit": 4.4375, "top_ids": [279, 2598, 3881, 809, 57340, 10769, 1083, 3371, 220, 264, 15503, 14576], "top_tokens": [" the", " called", " known", " Y", " yen", " Japanese", " also", " den", " ", " a", " primarily", " mainly"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 6.0, "target_rank": 8378.5, "source_logit": 20.0, "target_logit": 2.421875, "top_ids": [10769, 57340, 809, 619, 52948, 6323, 33554, 379, 13775, 138902, 17672, 51466], "top_tokens": [" Japanese", " yen", " Y", " J", " japanese", " Japan", " Yam", " y", " Ren", " Y\u00ean", " euro", "Japanese"]}], "source_token": 6323, "target_token": 6747, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Kyoto", " is", " the"], "seconds": 1.3653773330152035}
{"run_id": "eval-01/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0923035144805908, "target_logp": -20.100608825683594, "log_odds": -19.008305311203003, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.26544690132141113, -0.8268566727638245]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-18.693378448486328, -1.407179594039917, -5.173549288883805e-05]}}, "score_trace": {"source": [], "target": []}, "output": " Japanese Yen.\n\nFact: The currency used in the country", "output_ids": [10769, 809, 268, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [], "first_top_ids": [10769, 57340, 809, 619, 52948, 6323, 33554, 379, 13775, 138902], "first_top_logits": [28.125, 26.5, 25.875, 20.25, 20.25, 20.0, 19.0, 18.75, 18.625, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.511982582975179, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456354816, "mps_driver_bytes": 4004200448}}
{"run_id": "eval-01/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0923035144805908, "target_logp": -20.100608825683594, "log_odds": -19.008305311203003, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.26544690132141113, -0.8268566727638245]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-18.693378448486328, -1.407179594039917, -5.173549288883805e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese Yen.\n\nFact: The currency used in the country", "output_ids": [10769, 809, 268, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 57340, 809, 619, 52948, 6323, 33554, 379, 13775, 138902], "first_top_logits": [28.125, 26.5, 25.875, 20.25, 20.25, 20.0, 19.0, 18.75, 18.625, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.4967655837535858, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456387584, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -56.1319580078125, "target_logp": -8.2852783203125, "log_odds": 47.8466796875, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-54.00324249267578, -2.128714084625244]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-6.878068923950195, -1.4071576595306396, -5.1616290875244886e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.24567413330078, 50.44936752319336, 204.5745086669922, 123.16305541992188, 51.72624969482422, 56.42959976196289, 70.2757339477539, 263.94329833984375, 62.31447982788086, 847.6885986328125, 1257.73291015625, 1393.9798583984375]], "relative_norm": [[0.0031769080087542534, 0.06445679068565369, 0.17707890272140503, 0.10881222039461136, 0.04864209517836571, 0.05229194834828377, 0.06784715503454208, 0.26526904106140137, 0.06483383476734161, 0.8983335494995117, 1.111296534538269, 1.3679767847061157]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.007972717285156, 70.17706298828125, 259.8143615722656, 61.64262771606445, 850.37744140625, 1276.8870849609375, 1391.958251953125]], "actual_norm": [[50.24567413330078, 51.05774688720703, 205.45443725585938, 122.7387924194336, 53.28364181518555, 60.013092041015625, 70.15601348876953, 259.8038635253906, 61.641719818115234, 850.3818359375, 1276.90576171875, 1391.837890625]], "relative_norm": [[0.0031769080087542534, 0.06443841010332108, 0.17736217379570007, 0.1082456111907959, 0.050205688923597336, 0.0555141344666481, 0.06762254238128662, 0.26177728176116943, 0.06409899145364761, 0.8986847996711731, 1.137511968612671, 1.3666497468948364]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " India Rupee.  What is the currency used in the", "output_ids": [6747, 27914, 61498, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.24567413330078, 50.44936752319336, 204.5745086669922, 123.16305541992188, 51.72624969482422, 56.42959976196289, 70.2757339477539, 263.94329833984375, 62.31447982788086, 847.6885986328125, 1257.73291015625, 1393.9798583984375]], "relative_norm": [[0.0031769080087542534, 0.06445679068565369, 0.17707890272140503, 0.10881222039461136, 0.04864209517836571, 0.05229194834828377, 0.06784715503454208, 0.26526904106140137, 0.06483383476734161, 0.8983335494995117, 1.111296534538269, 1.3679767847061157]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6747, 7748, 330, 422, 364, 11, 304, 1565, 1119, 508], "first_top_logits": [27.25, 20.375, 19.375, 19.125, 19.125, 18.75, 18.25, 17.75, 17.375, 17.375], "source_correct": false, "target_correct": true, "seconds": 0.4512260830961168, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456387584, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -8.946610450744629, "target_logp": -20.601966857910156, "log_odds": -11.655356407165527, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-8.252145767211914, -0.694464385509491]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-19.377355575561523, -1.2244728803634644, -0.00014006110723130405]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.62623596191406, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.24561309814453, 50.3586311340332, 204.59434509277344, 123.21380615234375, 51.716033935546875, 56.40036392211914, 70.31730651855469, 263.93109130859375, 62.28840255737305, 847.6976318359375, 1257.8580322265625, 1394.0277099609375]], "relative_norm": [[0.003176904283463955, 0.06434085965156555, 0.17709606885910034, 0.10885705798864365, 0.048632487654685974, 0.052264854311943054, 0.06788729131221771, 0.26525676250457764, 0.06480669975280762, 0.8983430862426758, 1.1114070415496826, 1.3680237531661987]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.97592544555664, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.00797653198242, 70.17706298828125, 259.8143615722656, 61.64262390136719, 850.37744140625, 1276.8870849609375, 1391.958251953125]], "actual_norm": [[50.24561309814453, 50.96617126464844, 205.40565490722656, 122.7142562866211, 53.32664108276367, 60.02883529663086, 70.17925262451172, 259.76361083984375, 61.696598052978516, 850.361083984375, 1276.8656005859375, 1391.9444580078125]], "relative_norm": [[0.003176904283463955, 0.06432283669710159, 0.17732006311416626, 0.10822397470474243, 0.05024620145559311, 0.05552870035171509, 0.06764494627714157, 0.2617367208003998, 0.06415605545043945, 0.8986629247665405, 1.1374762058258057, 1.3667542934417725]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Kyoto", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 81022], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.62623596191406, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.24561309814453, 50.3586311340332, 204.59434509277344, 123.21380615234375, 51.716033935546875, 56.40036392211914, 70.31730651855469, 263.93109130859375, 62.28840255737305, 847.6976318359375, 1257.8580322265625, 1394.0277099609375]], "relative_norm": [[0.003176904283463955, 0.06434085965156555, 0.17709606885910034, 0.10885705798864365, 0.048632487654685974, 0.052264854311943054, 0.06788729131221771, 0.26525676250457764, 0.06480669975280762, 0.8983430862426758, 1.1114070415496826, 1.3680237531661987]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 6633, 10769, 659, 220, 57340, 5313, 3146, 198, 20742], "first_top_logits": [28.25, 21.5, 20.0, 19.125, 19.0, 18.25, 18.0, 17.75, 17.75, 17.625], "source_correct": false, "target_correct": false, "seconds": 0.4981059581041336, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9072470664978027, "target_logp": -42.74279022216797, "log_odds": -40.835543155670166, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.9323551058769226, -0.9748920202255249]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-41.428585052490234, -1.3141279220581055, -7.509902934543788e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.25120162963867, 50.38893508911133, 204.60997009277344, 123.22185516357422, 51.73271560668945, 56.456790924072266, 70.27601623535156, 263.96002197265625, 62.299835205078125, 847.856689453125, 1257.881591796875, 1393.90234375]], "relative_norm": [[0.003177257487550378, 0.06437958031892776, 0.17710959911346436, 0.10886417329311371, 0.0486481748521328, 0.052317146211862564, 0.06784743070602417, 0.265285849571228, 0.0648185983300209, 0.8985116481781006, 1.111427903175354, 1.3679007291793823]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.007972717285156, 70.17706298828125, 259.8143615722656, 61.64262771606445, 850.37744140625, 1276.8870849609375, 1391.958251953125]], "actual_norm": [[50.25120162963867, 51.053253173828125, 205.50123596191406, 122.75699615478516, 53.33518981933594, 60.011756896972656, 70.14012908935547, 259.7858581542969, 61.60143280029297, 850.4095458984375, 1276.8026123046875, 1391.7625732421875]], "relative_norm": [[0.003177257487550378, 0.06443274021148682, 0.1774025708436966, 0.10826166719198227, 0.05025425925850868, 0.05551290139555931, 0.06760723143815994, 0.2617591619491577, 0.06405709683895111, 0.8987141251564026, 1.1374200582504272, 1.3665757179260254]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japan yen.\n\nFact: The currency used in the country containing", "output_ids": [6323, 57340, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 847.7610473632812, 1257.828857421875, 1393.95947265625]], "actual_norm": [[50.25120162963867, 50.38893508911133, 204.60997009277344, 123.22185516357422, 51.73271560668945, 56.456790924072266, 70.27601623535156, 263.96002197265625, 62.299835205078125, 847.856689453125, 1257.881591796875, 1393.90234375]], "relative_norm": [[0.003177257487550378, 0.06437958031892776, 0.17710959911346436, 0.10886417329311371, 0.0486481748521328, 0.052317146211862564, 0.06784743070602417, 0.265285849571228, 0.0648185983300209, 0.8985116481781006, 1.111427903175354, 1.3679007291793823]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6323, 10769, 400, 619, 220, 57340, 353, 26970, 3693, 8248], "first_top_logits": [19.375, 19.125, 16.625, 15.9375, 15.5625, 14.9375, 14.75, 14.125, 13.5625, 13.5], "source_correct": true, "target_correct": false, "seconds": 0.48942791717126966, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.3224159479141235, "target_logp": -15.766854286193848, "log_odds": -14.444438338279724, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.34713420271873474, -0.9752817153930664]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-14.263850212097168, -1.502938151359558, -6.627816765103489e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.39919662475586, 204.62620544433594, 123.18102264404297, 51.68983840942383, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.8287353515625, 1393.9593505859375]], "actual_norm": [[50.17301940917969, 50.48334884643555, 204.52737426757812, 123.2200698852539, 51.707149505615234, 56.33024597167969, 70.3170166015625, 263.9566955566406, 62.28479766845703, 847.8264770507812, 1257.65576171875, 1393.9163818359375]], "relative_norm": [[0.0031723142601549625, 0.06450020521879196, 0.17703810334205627, 0.10886259377002716, 0.04862413555383682, 0.05219987779855728, 0.06788701564073563, 0.2652824819087982, 0.06480295211076736, 0.8984796404838562, 1.111228346824646, 1.3679144382476807]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.975921630859375, 205.4854278564453, 122.73771667480469, 53.34577178955078, 60.00796890258789, 70.17706298828125, 259.8143310546875, 61.64262390136719, 850.37744140625, 1276.8870849609375, 1391.958251953125]], "actual_norm": [[50.17301940917969, 50.99925231933594, 205.35479736328125, 122.75037384033203, 53.323455810546875, 59.951656341552734, 70.15821075439453, 259.78240966796875, 61.69449234008789, 850.4248657226562, 1276.93310546875, 1391.9044189453125]], "relative_norm": [[0.0031723142601549625, 0.06436458975076675, 0.177276149392128, 0.10825582593679428, 0.05024320259690285, 0.055457305163145065, 0.06762466579675674, 0.26175567507743835, 0.06415386497974396, 0.8987302780151367, 1.1375362873077393, 1.3667150735855103]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese Yen.  What is the currency used in the", "output_ids": [10769, 809, 268, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.39919662475586, 204.62620544433594, 123.18102264404297, 51.68983840942383, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.8287353515625, 1393.9593505859375]], "actual_norm": [[50.17301940917969, 50.48334884643555, 204.52737426757812, 123.2200698852539, 51.707149505615234, 56.33024597167969, 70.3170166015625, 263.9566955566406, 62.28479766845703, 847.8264770507812, 1257.65576171875, 1393.9163818359375]], "relative_norm": [[0.0031723142601549625, 0.06450020521879196, 0.17703810334205627, 0.10886259377002716, 0.04862413555383682, 0.05219987779855728, 0.06788701564073563, 0.2652824819087982, 0.06480295211076736, 0.8984796404838562, 1.111228346824646, 1.3679144382476807]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 17672, 220, 57340, 19461, 8585, 30, 619, 138902, 11413], "first_top_logits": [21.75, 20.125, 19.25, 18.5, 18.375, 18.0, 17.5, 16.25, 15.25, 15.25], "source_correct": true, "target_correct": false, "seconds": 0.4610132919624448, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.195838928222656, "target_logp": -21.78917694091797, "log_odds": -16.593338012695312, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-4.140778541564941, -1.0550601482391357]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-20.65111541748047, -1.1379750967025757, -8.523101132595912e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.18103790283203, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.92144775390625, 62.27316665649414, 847.7610473632812, 1257.828857421875, 1393.9595947265625]], "actual_norm": [[50.24762725830078, 50.41568374633789, 204.66673278808594, 123.2210922241211, 51.73438262939453, 56.42528533935547, 70.31700897216797, 263.91241455078125, 62.300045013427734, 847.7125854492188, 1257.830078125, 1393.9205322265625]], "relative_norm": [[0.0031770316418260336, 0.06441375613212585, 0.17715872824192047, 0.10886349529027939, 0.04864974319934845, 0.05228795111179352, 0.06788700819015503, 0.2652379870414734, 0.06481881439685822, 0.898358941078186, 1.1113823652267456, 1.367918610572815]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48545837402344, 122.73773193359375, 53.34577560424805, 60.00797653198242, 70.17707061767578, 259.8143615722656, 61.64263153076172, 850.3775024414062, 1276.8870849609375, 1391.9583740234375]], "actual_norm": [[50.24762725830078, 51.01027297973633, 205.51502990722656, 122.8192367553711, 53.32908248901367, 59.97407531738281, 70.20330047607422, 259.7947082519531, 61.64890670776367, 850.3798217773438, 1277.0328369140625, 1392.025390625]], "relative_norm": [[0.0031770316418260336, 0.06437849998474121, 0.17741447687149048, 0.10831655561923981, 0.05024850368499756, 0.0554780438542366, 0.0676681250333786, 0.26176807284355164, 0.06410646438598633, 0.898682713508606, 1.1376250982284546, 1.3668338060379028]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " which is the currency symbol.  What is the answer to", "output_ids": [892, 374, 279, 11413, 7735, 13, 220, 3555, 374, 279, 4226, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.18103790283203, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.92144775390625, 62.27316665649414, 847.7610473632812, 1257.828857421875, 1393.9595947265625]], "actual_norm": [[50.24762725830078, 50.41568374633789, 204.66673278808594, 123.2210922241211, 51.73438262939453, 56.42528533935547, 70.31700897216797, 263.91241455078125, 62.300045013427734, 847.7125854492188, 1257.830078125, 1393.9205322265625]], "relative_norm": [[0.0031770316418260336, 0.06441375613212585, 0.17715872824192047, 0.10886349529027939, 0.04864974319934845, 0.05228795111179352, 0.06788700819015503, 0.2652379870414734, 0.06481881439685822, 0.898358941078186, 1.1113823652267456, 1.367918610572815]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [892, 264, 13, 304, 29913, 11, 10769, 30, 17672, 6323], "first_top_logits": [17.625, 16.5, 16.25, 16.125, 15.75, 15.0, 14.5, 14.125, 14.125, 14.125], "source_correct": false, "target_correct": false, "seconds": 0.43313729111105204, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-01/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-01", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Kyoto", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Kyoto is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.5392122268676758, "target_logp": -16.072120666503906, "log_odds": -14.53290843963623, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.4841115176677704, -1.055100679397583]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-14.470783233642578, -1.6012638807296753, -7.509902934543788e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.39919662475586, 204.626220703125, 123.18102264404297, 51.689842224121094, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.8287353515625, 1393.95947265625]], "actual_norm": [[50.19971466064453, 50.403446197509766, 204.61605834960938, 123.24967193603516, 51.63630676269531, 56.389671325683594, 70.332763671875, 263.92413330078125, 62.275611877441406, 847.7763061523438, 1257.7401123046875, 1393.8389892578125]], "relative_norm": [[0.003174002282321453, 0.06439811736345291, 0.17711487412452698, 0.10888874530792236, 0.04855751618742943, 0.052254948765039444, 0.06790221482515335, 0.26524975895881653, 0.06479339301586151, 0.8984264731407166, 1.1113028526306152, 1.3678385019302368]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73771667480469, 53.34577178955078, 60.00796890258789, 70.17706298828125, 259.8143615722656, 61.64262390136719, 850.37744140625, 1276.8870849609375, 1391.958251953125]], "actual_norm": [[50.19971466064453, 50.96611404418945, 205.5232696533203, 122.7233657836914, 53.332672119140625, 59.94233703613281, 70.22904968261719, 259.90545654296875, 61.64547348022461, 850.3905029296875, 1276.9036865234375, 1391.9266357421875]], "relative_norm": [[0.003174002282321453, 0.06432276219129562, 0.17742158472537994, 0.10823200643062592, 0.05025188624858856, 0.055448684841394424, 0.06769294291734695, 0.26187965273857117, 0.0641028955578804, 0.8986939787864685, 1.137510061264038, 1.366736888885498]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese Yen.\n\nFact: The currency used in the country", "output_ids": [10769, 809, 268, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 81022, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.39919662475586, 204.626220703125, 123.18102264404297, 51.689842224121094, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 847.7610473632812, 1257.8287353515625, 1393.95947265625]], "actual_norm": [[50.19971466064453, 50.403446197509766, 204.61605834960938, 123.24967193603516, 51.63630676269531, 56.389671325683594, 70.332763671875, 263.92413330078125, 62.275611877441406, 847.7763061523438, 1257.7401123046875, 1393.8389892578125]], "relative_norm": [[0.003174002282321453, 0.06439811736345291, 0.17711487412452698, 0.10888874530792236, 0.04855751618742943, 0.052254948765039444, 0.06790221482515335, 0.26524975895881653, 0.06479339301586151, 0.8984264731407166, 1.1113028526306152, 1.3678385019302368]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 57340, 809, 6323, 19551, 386, 481, 220, 619, 13775], "first_top_logits": [19.25, 18.125, 17.375, 15.75, 14.9375, 14.4375, 14.4375, 14.4375, 14.1875, 14.125], "source_correct": true, "target_correct": false, "seconds": 0.4832483339123428, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/readout", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 33.5, "target_rank": 112.0, "source_logit": 6.40625, "target_logit": 5.09375, "top_ids": [8330, 38352, 15973, 7513, 45418, 6364, 3001, 3693, 11413, 13369, 1111, 10769], "top_tokens": [" Australia", "uenta", "currency", " European", ".currency", " English", "ica", " American", " currency", " Australian", "ule", " Japanese"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 16.0, "target_rank": 217.0, "source_logit": 7.71875, "target_logit": 4.65625, "top_ids": [15973, 11413, 96694, 53661, 3001, 27908, 28130, 18174, 4787, 3693, 126256, 83759], "top_tokens": ["currency", " currency", "onis", "cla", "ica", "ingo", "ija", "oret", " North", " American", "a\u0142", "_LA"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 2.0, "target_rank": 2251.0, "source_logit": 26.625, "target_logit": 11.125, "top_ids": [11413, 6323, 35602, 8330, 110839, 10769, 24357, 6489, 36445, 5837, 13369, 85466], "top_tokens": [" currency", " Japan", " currencies", " Australia", "\u5728\u65e5\u672c", " Japanese", " overseas", " international", " internationally", " countries", " Australian", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 1.0, "target_rank": 635.5, "source_logit": 20.875, "target_logit": 11.75, "top_ids": [6323, 11413, 7214, 65001, 37198, 10769, 87598, 47412, 74508, 8330, 105579, 12447], "top_tokens": [" Japan", " currency", " foreign", "\"\u2014", " Commonwealth", " Japanese", " canadian", " foreigners", "\u00bb.", " Australia", "\u6fb3\u5927\u5229\u4e9a", "[c"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 1304.5, "target_rank": 521.5, "source_logit": 2.0625, "target_logit": 3.125, "top_ids": [21251, 3776, 55964, 603, 56537, 5737, 7748, 7513, 28453, 29921, 12751, 3731], "top_tokens": ["onia", " One", "\u2011", "one", " _______,", " Group", " Indian", " European", " Currency", "\u2010", " Roman", " Red"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 78.5, "target_rank": 195.0, "source_logit": 5.53125, "target_logit": 4.5, "top_ids": [7179, 11766, 364, 479, 18174, 11413, 105469, 29991, 356, 362, 730, 3639], "top_tokens": [" International", " Standard", " '", " G", "oret", " currency", "\u79cd\u7c7b", "\u540d\u79f0", " C", " A", " K", " United"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 82.0, "target_rank": 1314.0, "source_logit": 16.75, "target_logit": 13.0625, "top_ids": [11413, 35602, 13867, 3146, 5313, 30176, 5837, 3283, 7214, 9867, 28453, 24357], "top_tokens": [" currency", " currencies", " county", " country", " national", " counties", " countries", " city", " foreign", " native", " Currency", " overseas"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 3.0, "target_rank": 604.0, "source_logit": 19.125, "target_logit": 13.0, "top_ids": [37198, 11413, 6323, 105492, 7214, 5313, 94936, 6489, 15072, 86049, 5837, 35602], "top_tokens": [" Commonwealth", " currency", " Japan", "\u4e2d\u534e\u4eba\u6c11", " foreign", " national", " denomination", " international", " Kingdom", " monarchy", " countries", " currencies"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 262.5, "target_rank": 1257.5, "source_logit": 4.0625, "target_logit": 1.984375, "top_ids": [3693, 4787, 89478, 279, 4229, 5434, 271, 111012, 1083, 11, 3639, 220], "top_tokens": [" American", " North", "\u2026\n", " the", " Ge", "\u2026\n\n", "\n\n", "\u4e16\u754c\u4e0a\u6700", " also", ",", " United", " "]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 200.5, "target_rank": 840.5, "source_logit": 4.78125, "target_logit": 2.828125, "top_ids": [1852, 3693, 5938, 4787, 4102, 3639, 271, 5055, 4882, 18174, 479, 362], "top_tokens": [" same", " American", " German", " North", "\u00a0", " United", "\n\n", " National", " South", "oret", " G", " A"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 816.0, "target_rank": 3482.0, "source_logit": 11.0, "target_logit": 8.0, "top_ids": [104142, 11413, 103408, 35602, 113883, 115160, 28453, 32488, 116103, 113596, 90475, 101237], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", "\u6b27\u5143", " currencies", "\u672c\u56fd", "\u5176\u4ed6\u56fd\u5bb6", " Currency", " euros", "\u666e\u901a\u8bdd", "\u56fd\u7c4d", "(currency", "\u7f8e\u5143"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 425.5, "target_rank": 13432.0, "source_logit": 6.1875, "target_logit": 0.0047607421875, "top_ids": [104142, 116866, 101237, 112397, 103408, 101406, 110065, 113883, 99599, 106333, 109442, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u7f8e\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u6b27\u5143", "\u8d27\u5e01", "\u6c47\u7387", "\u672c\u56fd", "\u56fd\u5bb6", "\u5916\u6c47", "\u8d22\u653f\u90e8", "\u6210\u5458\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 5.0, "target_rank": 9413.0, "source_logit": 22.125, "target_logit": 0.478515625, "top_ids": [10769, 11413, 11192, 57340, 6323, 17692, 19551, 104142, 28453, 32488, 51466, 80189], "top_tokens": [" Japanese", " currency", " dollars", " yen", " Japan", " dollar", " USD", "\u4eba\u6c11\u5e01", " Currency", " euros", "Japanese", " Dollars"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 21976.5, "source_logit": 24.625, "target_logit": -1.7109375, "top_ids": [10769, 6323, 57340, 51466, 11413, 101059, 17692, 104142, 19551, 11192, 26194, 28453], "top_tokens": [" Japanese", " Japan", " yen", "Japanese", " currency", "\u65e5\u672c", " dollar", "\u4eba\u6c11\u5e01", " USD", " dollars", " Tokyo", " Currency"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 15.5, "target_rank": 3288.5, "source_logit": 32.75, "target_logit": 8.1875, "top_ids": [11413, 104142, 11192, 35602, 17692, 28453, 80189, 101406, 57340, 10769, 101237, 103408], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " currencies", " dollar", " Currency", " Dollars", "\u8d27\u5e01", " yen", " Japanese", "\u7f8e\u5143", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 5.0, "target_rank": 24205.0, "source_logit": 26.875, "target_logit": -0.83203125, "top_ids": [104142, 10769, 101059, 57340, 6323, 11413, 101406, 35602, 101237, 51466, 17692, 110839], "top_tokens": ["\u4eba\u6c11\u5e01", " Japanese", "\u65e5\u672c", " yen", " Japan", " currency", "\u8d27\u5e01", " currencies", "\u7f8e\u5143", "Japanese", " dollar", "\u5728\u65e5\u672c"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 10.5, "target_rank": 7535.5, "source_logit": 20.5, "target_logit": 5.1875, "top_ids": [279, 3881, 10769, 57340, 2598, 809, 1083, 220, 2274, 1483, 6323, 619], "top_tokens": [" the", " known", " Japanese", " yen", " called", " Y", " also", " ", " US", " used", " Japan", " J"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 6264.5, "source_logit": 20.125, "target_logit": 3.0625, "top_ids": [10769, 57340, 809, 6323, 619, 52948, 33554, 51466, 17672, 379, 138902, 1532], "top_tokens": [" Japanese", " yen", " Y", " Japan", " J", " japanese", " Yam", "Japanese", " euro", " y", " Y\u00ean", " New"]}], "source_token": 6323, "target_token": 6747, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Osaka", " is", " the"], "seconds": 1.0846462501212955}
{"run_id": "eval-02/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.085997223854065, "target_logp": -19.342418670654297, "log_odds": -18.256421446800232, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.45273712277412415, -0.6332601308822632]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-18.028160095214844, -1.3142063617706299, -5.173549288883805e-05]}}, "score_trace": {"source": [], "target": []}, "output": " Japanese yen.  Inference: The currency used in the", "output_ids": [10769, 57340, 13, 220, 758, 2202, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [], "first_top_ids": [10769, 57340, 809, 6323, 619, 52948, 33554, 51466, 17672, 379], "first_top_logits": [27.375, 26.5, 25.5, 20.125, 19.875, 18.875, 18.75, 18.625, 18.625, 17.375], "source_correct": true, "target_correct": false, "seconds": 0.49758329102769494, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.085997223854065, "target_logp": -19.342418670654297, "log_odds": -18.256421446800232, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.45273712277412415, -0.6332601308822632]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-18.028160095214844, -1.3142063617706299, -5.173549288883805e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese yen.  Inference: The currency used in the", "output_ids": [10769, 57340, 13, 220, 758, 2202, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 57340, 809, 6323, 619, 52948, 33554, 51466, 17672, 379], "first_top_logits": [27.375, 26.5, 25.5, 20.125, 19.875, 18.875, 18.75, 18.625, 18.625, 17.375], "source_correct": true, "target_correct": false, "seconds": 0.46688166726380587, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -54.32114028930664, "target_logp": -8.070048332214355, "log_odds": 46.251091957092285, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-53.00665283203125, -1.314488410949707]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-6.755691051483154, -1.314299464225769, -5.8530047681415454e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24567413330078, 50.44936752319336, 204.5745086669922, 123.16305541992188, 51.72624969482422, 56.42959976196289, 70.2757339477539, 263.94329833984375, 62.31447982788086, 943.8914184570312, 1185.15625, 1465.759521484375]], "relative_norm": [[0.0031769080087542534, 0.06445679068565369, 0.17707890272140503, 0.10881222039461136, 0.04864209517836571, 0.05229194834828377, 0.06784715503454208, 0.26526904106140137, 0.06483383476734161, 0.969876229763031, 1.080031394958496, 1.4267494678497314]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.007972717285156, 70.17706298828125, 259.8143615722656, 61.64262771606445, 942.1148071289062, 1203.824951171875, 1481.955322265625]], "actual_norm": [[50.24567413330078, 51.05774688720703, 205.45443725585938, 122.7387924194336, 53.28364181518555, 60.013092041015625, 70.15601348876953, 259.8038635253906, 61.641719818115234, 942.0699462890625, 1203.763427734375, 1481.8665771484375]], "relative_norm": [[0.0031769080087542534, 0.06443841010332108, 0.17736217379570007, 0.1082456111907959, 0.050205688923597336, 0.0555141344666481, 0.06762254238128662, 0.26177728176116943, 0.06409899145364761, 0.9668623805046082, 1.0955253839492798, 1.4409457445144653]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " India Rupee.  What is the name of the country", "output_ids": [6747, 27914, 61498, 13, 220, 3555, 374, 279, 829, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24567413330078, 50.44936752319336, 204.5745086669922, 123.16305541992188, 51.72624969482422, 56.42959976196289, 70.2757339477539, 263.94329833984375, 62.31447982788086, 943.8914184570312, 1185.15625, 1465.759521484375]], "relative_norm": [[0.0031769080087542534, 0.06445679068565369, 0.17707890272140503, 0.10881222039461136, 0.04864209517836571, 0.05229194834828377, 0.06784715503454208, 0.26526904106140137, 0.06483383476734161, 0.969876229763031, 1.080031394958496, 1.4267494678497314]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6747, 422, 7748, 330, 304, 11, 364, 1119, 27914, 1565], "first_top_logits": [26.375, 20.125, 19.875, 18.25, 18.125, 18.125, 17.875, 17.375, 17.25, 17.25], "source_correct": false, "target_correct": true, "seconds": 0.4992425423115492, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -8.931375503540039, "target_logp": -21.35169219970703, "log_odds": -12.420316696166992, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-8.502001762390137, -0.42937421798706055]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-20.127182006835938, -1.2243857383728027, -0.00012361239350866526]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.62623596191406, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24561309814453, 50.3586311340332, 204.59434509277344, 123.21380615234375, 51.716033935546875, 56.40036392211914, 70.31730651855469, 263.93109130859375, 62.28840255737305, 943.8306274414062, 1185.2581787109375, 1465.594482421875]], "relative_norm": [[0.003176904283463955, 0.06434085965156555, 0.17709606885910034, 0.10885705798864365, 0.048632487654685974, 0.052264854311943054, 0.06788729131221771, 0.26525676250457764, 0.06480669975280762, 0.9698137640953064, 1.0801243782043457, 1.426588773727417]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.97592544555664, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.00797653198242, 70.17706298828125, 259.8143615722656, 61.64262390136719, 942.1148681640625, 1203.824951171875, 1481.955322265625]], "actual_norm": [[50.24561309814453, 50.96617126464844, 205.40565490722656, 122.7142562866211, 53.32664108276367, 60.02883529663086, 70.17925262451172, 259.76361083984375, 61.696598052978516, 942.1027221679688, 1203.8106689453125, 1481.8582763671875]], "relative_norm": [[0.003176904283463955, 0.06432283669710159, 0.17732006311416626, 0.10822397470474243, 0.05024620145559311, 0.05552870035171509, 0.06764494627714157, 0.2617367208003998, 0.06415605545043945, 0.9668959975242615, 1.095568299293518, 1.4409377574920654]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Osaka", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 86985], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.62623596191406, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24561309814453, 50.3586311340332, 204.59434509277344, 123.21380615234375, 51.716033935546875, 56.40036392211914, 70.31730651855469, 263.93109130859375, 62.28840255737305, 943.8306274414062, 1185.2581787109375, 1465.594482421875]], "relative_norm": [[0.003176904283463955, 0.06434085965156555, 0.17709606885910034, 0.10885705798864365, 0.048632487654685974, 0.052264854311943054, 0.06788729131221771, 0.26525676250457764, 0.06480669975280762, 0.9698137640953064, 1.0801243782043457, 1.426588773727417]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 6633, 10769, 220, 5313, 659, 57340, 19551, 20742, 11413], "first_top_logits": [28.75, 22.125, 20.25, 19.25, 19.0, 18.25, 18.125, 18.0, 17.75, 17.5], "source_correct": false, "target_correct": false, "seconds": 0.4826934589073062, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.6792945861816406, "target_logp": -44.288414001464844, "log_odds": -42.6091194152832, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-1.1028543710708618, -0.5764402747154236]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-42.974185943603516, -1.3141648769378662, -6.627816765103489e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.25120162963867, 50.38893508911133, 204.60997009277344, 123.22185516357422, 51.73271560668945, 56.456790924072266, 70.27601623535156, 263.96002197265625, 62.299835205078125, 943.8642578125, 1185.4432373046875, 1465.513916015625]], "relative_norm": [[0.003177257487550378, 0.06437958031892776, 0.17710959911346436, 0.10886417329311371, 0.0486481748521328, 0.052317146211862564, 0.06784743070602417, 0.265285849571228, 0.0648185983300209, 0.9698483347892761, 1.0802929401397705, 1.4265103340148926]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73772430419922, 53.34577560424805, 60.007972717285156, 70.17706298828125, 259.8143615722656, 61.64262771606445, 942.1148071289062, 1203.824951171875, 1481.955322265625]], "actual_norm": [[50.25120162963867, 51.053253173828125, 205.50123596191406, 122.75699615478516, 53.33518981933594, 60.011756896972656, 70.14012908935547, 259.7858581542969, 61.60143280029297, 942.0872802734375, 1203.7503662109375, 1481.9615478515625]], "relative_norm": [[0.003177257487550378, 0.06443274021148682, 0.1774025708436966, 0.10826166719198227, 0.05025425925850868, 0.05551290139555931, 0.06760723143815994, 0.2617591619491577, 0.06405709683895111, 0.9668801426887512, 1.0955134630203247, 1.4410381317138672]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japan yen.\n\nFact: The currency used in the country containing", "output_ids": [6323, 57340, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.1810302734375, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.9214172363281, 62.273162841796875, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.25120162963867, 50.38893508911133, 204.60997009277344, 123.22185516357422, 51.73271560668945, 56.456790924072266, 70.27601623535156, 263.96002197265625, 62.299835205078125, 943.8642578125, 1185.4432373046875, 1465.513916015625]], "relative_norm": [[0.003177257487550378, 0.06437958031892776, 0.17710959911346436, 0.10886417329311371, 0.0486481748521328, 0.052317146211862564, 0.06784743070602417, 0.265285849571228, 0.0648185983300209, 0.9698483347892761, 1.0802929401397705, 1.4265103340148926]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6323, 10769, 619, 400, 57340, 220, 353, 26970, 3693, 29200], "first_top_logits": [19.75, 19.125, 16.0, 15.125, 14.8125, 14.75, 13.375, 13.1875, 13.1875, 13.0], "source_correct": true, "target_correct": false, "seconds": 0.5001884582452476, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.492025375366211, "target_logp": -15.596473693847656, "log_odds": -14.104448318481445, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.9681028723716736, -0.5239225625991821]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-14.18927001953125, -1.407128095626831, -7.521823135903105e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.39919662475586, 204.62620544433594, 123.18102264404297, 51.68983840942383, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.17301940917969, 50.48334884643555, 204.52737426757812, 123.2200698852539, 51.707149505615234, 56.33024597167969, 70.3170166015625, 263.9566955566406, 62.28479766845703, 943.8793334960938, 1185.2606201171875, 1465.745361328125]], "relative_norm": [[0.0031723142601549625, 0.06450020521879196, 0.17703810334205627, 0.10886259377002716, 0.04862413555383682, 0.05219987779855728, 0.06788701564073563, 0.2652824819087982, 0.06480295211076736, 0.969863772392273, 1.0801265239715576, 1.4267356395721436]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.975921630859375, 205.4854278564453, 122.73771667480469, 53.34577178955078, 60.00796890258789, 70.17706298828125, 259.8143310546875, 61.64262390136719, 942.11474609375, 1203.8248291015625, 1481.955322265625]], "actual_norm": [[50.17301940917969, 50.99925231933594, 205.35479736328125, 122.75037384033203, 53.323455810546875, 59.951656341552734, 70.15821075439453, 259.78240966796875, 61.69449234008789, 942.2418212890625, 1203.88818359375, 1481.89013671875]], "relative_norm": [[0.0031723142601549625, 0.06436458975076675, 0.177276149392128, 0.10825582593679428, 0.05024320259690285, 0.055457305163145065, 0.06762466579675674, 0.26175567507743835, 0.06415386497974396, 0.9670387506484985, 1.0956388711929321, 1.4409687519073486]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  What is the currency used in the country containing", "output_ids": [17672, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.27054214477539, 50.39919662475586, 204.62620544433594, 123.18102264404297, 51.68983840942383, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.17301940917969, 50.48334884643555, 204.52737426757812, 123.2200698852539, 51.707149505615234, 56.33024597167969, 70.3170166015625, 263.9566955566406, 62.28479766845703, 943.8793334960938, 1185.2606201171875, 1465.745361328125]], "relative_norm": [[0.0031723142601549625, 0.06450020521879196, 0.17703810334205627, 0.10886259377002716, 0.04862413555383682, 0.05219987779855728, 0.06788701564073563, 0.2652824819087982, 0.06480295211076736, 0.969863772392273, 1.0801265239715576, 1.4267356395721436]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 10769, 57340, 220, 19461, 8585, 30, 619, 32488, 103408], "first_top_logits": [21.375, 21.125, 18.75, 18.625, 18.625, 17.375, 17.25, 15.6875, 15.5625, 15.375], "source_correct": false, "target_correct": false, "seconds": 0.48178266594186425, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.095304489135742, "target_logp": -21.759723663330078, "log_odds": -16.664419174194336, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-4.336867332458496, -0.758436918258667]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-20.621835708618164, -1.1378138065338135, -7.521823135903105e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.18103790283203, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.92144775390625, 62.27316665649414, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24762725830078, 50.41568374633789, 204.66673278808594, 123.2210922241211, 51.73438262939453, 56.42528533935547, 70.31700897216797, 263.91241455078125, 62.300045013427734, 944.0715942382812, 1185.1705322265625, 1465.607177734375]], "relative_norm": [[0.0031770316418260336, 0.06441375613212585, 0.17715872824192047, 0.10886349529027939, 0.04864974319934845, 0.05228795111179352, 0.06788700819015503, 0.2652379870414734, 0.06481881439685822, 0.9700613617897034, 1.0800443887710571, 1.4266011714935303]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48545837402344, 122.73773193359375, 53.34577560424805, 60.00797653198242, 70.17707061767578, 259.8143615722656, 61.64263153076172, 942.1148681640625, 1203.8250732421875, 1481.955322265625]], "actual_norm": [[50.24762725830078, 51.01027297973633, 205.51502990722656, 122.8192367553711, 53.32908248901367, 59.97407531738281, 70.20330047607422, 259.7947082519531, 61.64890670776367, 942.1107177734375, 1203.84716796875, 1482.1322021484375]], "relative_norm": [[0.0031770316418260336, 0.06437849998474121, 0.17741447687149048, 0.10831655561923981, 0.05024850368499756, 0.0554780438542366, 0.0676681250333786, 0.26176807284355164, 0.06410646438598633, 0.9669042229652405, 1.0956015586853027, 1.4412040710449219]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " which is the currency symbol.  What is the answer to", "output_ids": [892, 374, 279, 11413, 7735, 13, 220, 3555, 374, 279, 4226, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.399200439453125, 204.626220703125, 123.18103790283203, 51.689842224121094, 56.37942886352539, 70.31771850585938, 263.92144775390625, 62.27316665649414, 943.9300537109375, 1185.238525390625, 1465.7325439453125]], "actual_norm": [[50.24762725830078, 50.41568374633789, 204.66673278808594, 123.2210922241211, 51.73438262939453, 56.42528533935547, 70.31700897216797, 263.91241455078125, 62.300045013427734, 944.0715942382812, 1185.1705322265625, 1465.607177734375]], "relative_norm": [[0.0031770316418260336, 0.06441375613212585, 0.17715872824192047, 0.10886349529027939, 0.04864974319934845, 0.05228795111179352, 0.06788700819015503, 0.2652379870414734, 0.06481881439685822, 0.9700613617897034, 1.0800443887710571, 1.4266011714935303]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [892, 264, 29913, 13, 304, 30, 17672, 11, 31275, 10769], "first_top_logits": [16.625, 16.0, 16.0, 15.625, 15.5, 14.125, 14.0625, 14.0, 13.8125, 13.75], "source_correct": false, "target_correct": false, "seconds": 0.467290457803756, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-02/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-02", "split": "eval", "country": "Japan", "currency": "Japanese yen", "aliases": ["yen", "JPY"], "city": "Osaka", "target_country": "India", "target_currency": "Indian rupee", "target_aliases": ["rupee", "INR"], "prompt": "Fact: The currency used in the country containing Osaka is the", "source_continuation": " Japanese yen", "target_continuation": " Indian rupee"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2970927953720093, "target_logp": -16.471437454223633, "log_odds": -15.174344658851624, "token_scores": {"source": {"ids": [10769, 57340], "log_probs": [-0.6032207608222961, -0.6938720345497131]}, "target": {"ids": [7748, 11122, 61498], "log_probs": [-14.769158363342285, -1.7021942138671875, -8.523101132595912e-05]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.39919662475586, 204.626220703125, 123.18102264404297, 51.689842224121094, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.732421875]], "actual_norm": [[50.19971466064453, 50.403446197509766, 204.61605834960938, 123.24967193603516, 51.63630676269531, 56.389671325683594, 70.332763671875, 263.92413330078125, 62.275611877441406, 943.9788208007812, 1184.95849609375, 1465.75146484375]], "relative_norm": [[0.003174002282321453, 0.06439811736345291, 0.17711487412452698, 0.10888874530792236, 0.04855751618742943, 0.052254948765039444, 0.06790221482515335, 0.26524975895881653, 0.06479339301586151, 0.9699660539627075, 1.0798512697219849, 1.426741600036621]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.975921630859375, 205.48544311523438, 122.73771667480469, 53.34577178955078, 60.00796890258789, 70.17706298828125, 259.8143615722656, 61.64262390136719, 942.11474609375, 1203.824951171875, 1481.955322265625]], "actual_norm": [[50.19971466064453, 50.96611404418945, 205.5232696533203, 122.7233657836914, 53.332672119140625, 59.94233703613281, 70.22904968261719, 259.90545654296875, 61.64547348022461, 942.0455322265625, 1203.855712890625, 1481.938232421875]], "relative_norm": [[0.003174002282321453, 0.06432276219129562, 0.17742158472537994, 0.10823200643062592, 0.05025188624858856, 0.055448684841394424, 0.06769294291734695, 0.26187965273857117, 0.0641028955578804, 0.9668373465538025, 1.0956093072891235, 1.4410154819488525]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese Yen.\n\nFact: The currency used in the country", "output_ids": [10769, 809, 268, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 86985, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[50.270545959472656, 50.39919662475586, 204.626220703125, 123.18102264404297, 51.689842224121094, 56.379425048828125, 70.31771850585938, 263.9214172363281, 62.27315902709961, 943.9299926757812, 1185.238525390625, 1465.732421875]], "actual_norm": [[50.19971466064453, 50.403446197509766, 204.61605834960938, 123.24967193603516, 51.63630676269531, 56.389671325683594, 70.332763671875, 263.92413330078125, 62.275611877441406, 943.9788208007812, 1184.95849609375, 1465.75146484375]], "relative_norm": [[0.003174002282321453, 0.06439811736345291, 0.17711487412452698, 0.10888874530792236, 0.04855751618742943, 0.052254948765039444, 0.06790221482515335, 0.26524975895881653, 0.06479339301586151, 0.9699660539627075, 1.0798512697219849, 1.426741600036621]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 57340, 809, 19551, 6323, 481, 386, 17692, 547, 619], "first_top_logits": [18.875, 17.875, 17.25, 16.375, 16.0, 14.75, 14.3125, 14.1875, 14.0, 13.75], "source_correct": true, "target_correct": false, "seconds": 0.49909862503409386, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/readout", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 8.0, "target_rank": 1046.5, "source_logit": 7.65625, "target_logit": 2.484375, "top_ids": [7748, 8330, 38352, 45418, 1111, 15973, 7065, 6747, 11433, 47238, 3001, 43393], "top_tokens": [" Indian", " Australia", "uenta", ".currency", "ule", "currency", "ican", " India", "arry", " Regulation", "ica", " Generally"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 45.0, "target_rank": 398.0, "source_logit": 6.375, "target_logit": 3.796875, "top_ids": [96694, 15973, 11413, 53661, 27908, 3693, 3001, 42189, 5088, 11496, 28130, 4787], "top_tokens": ["onis", "currency", " currency", "cla", "ingo", " American", "ica", "etz", "urrency", "ika", "ija", " North"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 46.5, "target_rank": 142.5, "source_logit": 19.5, "target_logit": 17.5, "top_ids": [11413, 12463, 35602, 3263, 8330, 3146, 7855, 5837, 6489, 22956, 13369, 7214], "top_tokens": [" currency", " colour", " currencies", "\".", " Australia", " country", " British", " countries", " international", "\".\n", " Australian", " foreign"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 12.0, "target_rank": 70.5, "source_logit": 15.9375, "target_logit": 14.0625, "top_ids": [11413, 7855, 7214, 37198, 7748, 65001, 13224, 3220, 2293, 5313, 3146, 6747], "top_tokens": [" currency", " British", " foreign", " Commonwealth", " Indian", "\"\u2014", " Britain", " money", "\u2014", " national", " country", " India"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 47.0, "target_rank": 1249.5, "source_logit": 5.59375, "target_logit": 2.140625, "top_ids": [7748, 21251, 3776, 603, 28453, 5737, 56537, 32671, 12751, 1200, 3731, 24355], "top_tokens": [" Indian", "onia", " One", "one", " Currency", " Group", " _______,", " ______", " Roman", " Con", " Red", " Medium"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 17.5, "target_rank": 108.5, "source_logit": 6.625, "target_logit": 4.9375, "top_ids": [7179, 364, 11413, 7748, 479, 11766, 21251, 547, 362, 356, 467, 3639], "top_tokens": [" International", " '", " currency", " Indian", " G", " Standard", "onia", " U", " A", " C", " W", " United"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 46.5, "target_rank": 377.5, "source_logit": 16.25, "target_logit": 14.0, "top_ids": [11413, 35602, 3146, 5313, 13867, 29973, 5837, 22956, 91984, 12463, 58243, 3283], "top_tokens": [" currency", " currencies", " country", " national", " county", " licence", " countries", "\".\n", " licences", " colour", " nationality", " city"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 26.5, "target_rank": 110.5, "source_logit": 16.125, "target_logit": 14.375, "top_ids": [37198, 11413, 7855, 5313, 6364, 7748, 94936, 3146, 13867, 7214, 5837, 5297], "top_tokens": [" Commonwealth", " currency", " British", " national", " English", " Indian", " denomination", " country", " county", " foreign", " countries", " standard"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 249.0, "target_rank": 1233.0, "source_logit": 4.25, "target_logit": 2.078125, "top_ids": [3693, 4787, 89478, 4229, 2240, 4102, 5434, 6364, 271, 46427, 3639, 279], "top_tokens": [" American", " North", "\u2026\n", " Ge", " None", "\u00a0", "\u2026\n\n", " English", "\n\n", "/is", " United", " the"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 213.0, "target_rank": 187.0, "source_logit": 4.5625, "target_logit": 4.75, "top_ids": [1852, 3693, 3639, 5055, 5938, 7179, 271, 4102, 4787, 4882, 6364, 2701], "top_tokens": [" same", " American", " United", " National", " German", " International", "\n\n", "\u00a0", " North", " South", " English", " following"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 101.0, "target_rank": 733.0, "source_logit": 16.75, "target_logit": 12.9375, "top_ids": [11413, 104142, 35602, 28453, 103408, 113883, 115160, 106333, 90475, 26321, 113596, 77186], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " Currency", "\u6b27\u5143", "\u672c\u56fd", "\u5176\u4ed6\u56fd\u5bb6", "\u5916\u6c47", "(currency", "Currency", "\u56fd\u7c4d", ".Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 732.5, "target_rank": 2134.0, "source_logit": 8.5, "target_logit": 6.1875, "top_ids": [104142, 101237, 106333, 11413, 101406, 112397, 116866, 109442, 110065, 113883, 35602, 28453], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", "\u5916\u6c47", " currency", "\u8d27\u5e01", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u8d22\u653f\u90e8", "\u6c47\u7387", "\u672c\u56fd", " currencies", " Currency"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 21.5, "target_rank": 443.5, "source_logit": 17.25, "target_logit": 6.875, "top_ids": [11413, 104142, 11192, 28453, 112455, 17692, 19551, 7748, 77186, 80189, 35602, 90475], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " Currency", "\u82f1\u9551", " dollar", " USD", " Indian", ".Currency", " Dollars", " currencies", "(currency"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 7.0, "target_rank": 299.5, "source_logit": 18.75, "target_logit": 7.34375, "top_ids": [11413, 104142, 28453, 7748, 17692, 35602, 6747, 11192, 19551, 3220, 101406, 7855], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " Currency", " Indian", " dollar", " currencies", " India", " dollars", " USD", " money", "\u8d27\u5e01", " British"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 26.5, "target_rank": 1396.0, "source_logit": 28.375, "target_logit": 10.3125, "top_ids": [104142, 11413, 28453, 35602, 11192, 112455, 101237, 80189, 101406, 77186, 17692, 103408], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Currency", " currencies", " dollars", "\u82f1\u9551", "\u7f8e\u5143", " Dollars", "\u8d27\u5e01", ".Currency", " dollar", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 25.0, "target_rank": 1793.5, "source_logit": 24.375, "target_logit": 7.125, "top_ids": [104142, 11413, 28453, 35602, 101406, 101237, 77186, 26321, 17692, 112455, 45418, 15973], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Currency", " currencies", "\u8d27\u5e01", "\u7f8e\u5143", ".Currency", "Currency", " dollar", "\u82f1\u9551", ".currency", "currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 10.0, "target_rank": 3698.0, "source_logit": 21.125, "target_logit": 7.03125, "top_ids": [279, 27914, 11122, 7748, 2598, 78252, 3881, 7855, 1964, 6747, 2263, 2672], "top_tokens": [" the", " Ru", " ru", " Indian", " called", " Pound", " known", " British", " IN", " India", " Ind", "ru"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 12024.0, "source_logit": 22.25, "target_logit": 2.671875, "top_ids": [7748, 11122, 27914, 6747, 34712, 1852, 78252, 2263, 5313, 42225, 47564, 7855], "top_tokens": [" Indian", " ru", " Ru", " India", " Mumbai", " same", " Pound", " Ind", " national", " indian", "Indian", " British"]}], "source_token": 6747, "target_token": 5616, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Mumbai", " is", " the"], "seconds": 1.0839547077193856}
{"run_id": "eval-03/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.3071690797805786, "target_logp": -20.559459686279297, "log_odds": -19.25229060649872, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.16966454684734344, -1.1374727487564087, -3.182837463100441e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-20.466747283935547, -0.09271174669265747]}}, "score_trace": {"source": [], "target": []}, "output": " Indian Rupee.  In the country containing Mumbai, the", "output_ids": [7748, 27914, 61498, 13, 220, 758, 279, 3146, 8482, 34712, 11, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [], "first_top_ids": [7748, 11122, 27914, 6747, 34712, 1852, 78252, 2263, 5313, 42225], "first_top_logits": [29.875, 27.875, 26.375, 22.25, 21.0, 20.875, 20.25, 20.125, 20.0, 19.875], "source_correct": true, "target_correct": false, "seconds": 0.4987506251782179, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.3071690797805786, "target_logp": -20.559459686279297, "log_odds": -19.25229060649872, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.16966454684734344, -1.1374727487564087, -3.182837463100441e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-20.466747283935547, -0.09271174669265747]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian Rupee.  In the country containing Mumbai, the", "output_ids": [7748, 27914, 61498, 13, 220, 758, 279, 3146, 8482, 34712, 11, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 11122, 27914, 6747, 34712, 1852, 78252, 2263, 5313, 42225], "first_top_logits": [29.875, 27.875, 26.375, 22.25, 21.0, 20.875, 20.25, 20.125, 20.0, 19.875], "source_correct": true, "target_correct": false, "seconds": 0.49945174995809793, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -34.569515228271484, "target_logp": -0.13345102965831757, "log_odds": 34.43606419861317, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-33.59464645385742, -0.9748348593711853, -3.611976353568025e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-0.06209522858262062, -0.07135580480098724]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.2892608642578, 715.0905151367188, 931.8616333007812, 931.2835083007812]], "actual_norm": [[82.7910385131836, 147.4210662841797, 43.446006774902344, 33.4132194519043, 200.92709350585938, 145.02938842773438, 92.1517333984375, 132.08673095703125, 170.29197692871094, 715.1386108398438, 931.9542846679688, 931.3176879882812]], "relative_norm": [[0.005234669893980026, 0.18605560064315796, 0.037505533546209335, 0.029467737302184105, 0.18932044506072998, 0.13415707647800446, 0.08882395923137665, 0.1330900341272354, 0.1770804524421692, 0.7419588565826416, 0.8758834004402161, 0.8883945345878601]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108032226562, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.7910385131836, 144.49790954589844, 45.82680130004883, 29.602994918823242, 197.79165649414062, 150.53582763671875, 94.48457336425781, 127.19356536865234, 173.55307006835938, 718.0330200195312, 929.35205078125, 936.0016479492188]], "relative_norm": [[0.005234669893980026, 0.18461820483207703, 0.03966750204563141, 0.02615368366241455, 0.18599842488765717, 0.13949792087078094, 0.09121939539909363, 0.12783242762088776, 0.18056976795196533, 0.7463753819465637, 0.8726020455360413, 0.8936885595321655]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Chinese yuan.  What is the currency used in the country", "output_ids": [8453, 72283, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108032226562, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.7910385131836, 144.49790954589844, 45.82680130004883, 29.602994918823242, 197.79165649414062, 150.53582763671875, 94.48457336425781, 127.19356536865234, 173.55307006835938, 718.0330200195312, 929.35205078125, 936.0016479492188]], "relative_norm": [[0.005234669893980026, 0.18461820483207703, 0.03966750204563141, 0.02615368366241455, 0.18599842488765717, 0.13949792087078094, 0.09121939539909363, 0.12783242762088776, 0.18056976795196533, 0.7463753819465637, 0.8726020455360413, 0.8936885595321655]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8453, 13775, 8853, 5616, 431, 356, 2274, 5672, 72283, 220], "first_top_logits": [29.875, 26.125, 25.5, 25.0, 24.75, 24.5, 24.375, 23.25, 22.625, 22.375], "source_correct": false, "target_correct": true, "seconds": 0.48346804128959775, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.5626938343048096, "target_logp": -20.73670768737793, "log_odds": -17.17401385307312, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-2.58787202835083, -0.9747546911239624, -6.69933797325939e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-20.671615600585938, -0.06509262323379517]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.0554656982422, 92.1599349975586, 132.1023712158203, 170.28924560546875, 715.0905151367188, 931.8616333007812, 931.2835083007812]], "actual_norm": [[82.76513671875, 147.4741973876953, 43.41781234741211, 33.481876373291016, 200.99444580078125, 145.09686279296875, 92.20323944091797, 132.1643524169922, 170.2794647216797, 715.1407470703125, 931.9024047851562, 931.2328491210938]], "relative_norm": [[0.005233032163232565, 0.18612265586853027, 0.037481192499399185, 0.029528286308050156, 0.18938390910625458, 0.13421949744224548, 0.08887360244989395, 0.13316825032234192, 0.17706744372844696, 0.7419610619544983, 0.8758346438407898, 0.8883135914802551]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.806968688964844, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108642578125, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.76513671875, 144.47010803222656, 45.829097747802734, 29.56659698486328, 197.77349853515625, 150.56236267089844, 94.4558334350586, 127.17008209228516, 173.4987335205078, 718.09130859375, 929.34814453125, 936.033203125]], "relative_norm": [[0.005233032163232565, 0.1845826804637909, 0.03966948762536049, 0.026121526956558228, 0.18598134815692902, 0.1395225077867508, 0.09119164943695068, 0.1278088241815567, 0.18051323294639587, 0.7464359402656555, 0.8725983500480652, 0.8937187194824219]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Mumbai", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 34712], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.806968688964844, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108642578125, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.76513671875, 144.47010803222656, 45.829097747802734, 29.56659698486328, 197.77349853515625, 150.56236267089844, 94.4558334350586, 127.17008209228516, 173.4987335205078, 718.09130859375, 929.34814453125, 936.033203125]], "relative_norm": [[0.005233032163232565, 0.1845826804637909, 0.03966948762536049, 0.026121526956558228, 0.18598134815692902, 0.1395225077867508, 0.09119164943695068, 0.1278088241815567, 0.18051323294639587, 0.7464359402656555, 0.8725983500480652, 0.8937187194824219]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 7748, 6633, 11122, 1852, 1257, 6747, 220, 3146, 13], "first_top_logits": [26.875, 24.25, 20.75, 20.25, 19.375, 19.125, 19.0, 18.75, 18.75, 18.625], "source_correct": false, "target_correct": false, "seconds": 0.4829891249537468, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2916903495788574, "target_logp": -45.15886688232422, "log_odds": -43.86717653274536, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.06770715862512589, -1.223936915397644, -4.625213477993384e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-45.06578063964844, -0.09308718144893646]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.2892608642578, 715.0905151367188, 931.8616333007812, 931.2835083007812]], "actual_norm": [[82.79634094238281, 147.4833526611328, 43.45363235473633, 33.430023193359375, 200.9598846435547, 145.03907775878906, 92.17308044433594, 132.0720672607422, 170.34939575195312, 715.0964965820312, 931.806640625, 931.385986328125]], "relative_norm": [[0.005235005170106888, 0.1861342042684555, 0.03751211613416672, 0.029482556506991386, 0.18935135006904602, 0.13416604697704315, 0.08884453028440475, 0.1330752670764923, 0.17714016139507294, 0.7419151663780212, 0.875744640827179, 0.8884596824645996]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108032226562, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.79634094238281, 144.5224609375, 45.831748962402344, 29.62543487548828, 197.7605438232422, 150.51473999023438, 94.4526138305664, 127.14608764648438, 173.5244598388672, 718.088623046875, 929.2059326171875, 935.9735107421875]], "relative_norm": [[0.005235005170106888, 0.18464957177639008, 0.03967178612947464, 0.026173509657382965, 0.18596915900707245, 0.13947837054729462, 0.09118853509426117, 0.12778471410274506, 0.18053999543190002, 0.7464331984519958, 0.8724648356437683, 0.8936616778373718]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian Rupee.  What is the currency of the country", "output_ids": [7748, 27914, 61498, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 718.0108032226562, 929.343505859375, 936.0150146484375]], "actual_norm": [[82.79634094238281, 144.5224609375, 45.831748962402344, 29.62543487548828, 197.7605438232422, 150.51473999023438, 94.4526138305664, 127.14608764648438, 173.5244598388672, 718.088623046875, 929.2059326171875, 935.9735107421875]], "relative_norm": [[0.005235005170106888, 0.18464957177639008, 0.03967178612947464, 0.026173509657382965, 0.18596915900707245, 0.13947837054729462, 0.09118853509426117, 0.12778471410274506, 0.18053999543190002, 0.7464331984519958, 0.8724648356437683, 0.8936616778373718]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 6747, 11122, 1257, 11413, 27914, 2263, 198, 9145, 1852], "first_top_logits": [20.5, 17.5, 15.6875, 13.75, 13.375, 13.25, 13.0625, 12.75, 12.625, 12.5625], "source_correct": true, "target_correct": false, "seconds": 0.4481781250797212, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2544450759887695, "target_logp": -16.695980072021484, "log_odds": -15.441534996032715, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.030306274071335793, -1.2241066694259644, -3.218599158572033e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-16.59035301208496, -0.10562799125909805]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45849609375, 43.41843032836914, 33.43308639526367, 200.94480895996094, 145.05545043945312, 92.15992736816406, 132.1023712158203, 170.28924560546875, 715.0904541015625, 931.861572265625, 931.283447265625]], "actual_norm": [[82.77973937988281, 147.47142028808594, 43.46282196044922, 33.45431900024414, 201.03973388671875, 145.03663635253906, 92.27825164794922, 132.09547424316406, 170.27691650390625, 715.1511840820312, 931.9559936523438, 931.3116455078125]], "relative_norm": [[0.005233955569565296, 0.18611913919448853, 0.03752004727721214, 0.02950398251414299, 0.18942658603191376, 0.1341637820005417, 0.0889459103345871, 0.13309885561466217, 0.17706479132175446, 0.7419719099998474, 0.875885009765625, 0.8883887529373169]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.7652130126953, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 718.0107421875, 929.3434448242188, 936.0149536132812]], "actual_norm": [[82.77973937988281, 144.49253845214844, 45.828460693359375, 29.598468780517578, 197.741943359375, 150.55831909179688, 94.59871673583984, 127.08689880371094, 173.58749389648438, 717.9981079101562, 929.3573608398438, 935.9592895507812]], "relative_norm": [[0.005233955569565296, 0.18461133539676666, 0.039668936282396317, 0.026149684563279152, 0.18595166504383087, 0.13951876759529114, 0.09132958948612213, 0.12772522866725922, 0.18060557544231415, 0.7463390827178955, 0.8726069927215576, 0.8936481475830078]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian Rupee.  What is the currency of the country", "output_ids": [7748, 27914, 61498, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.7652130126953, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 718.0107421875, 929.3434448242188, 936.0149536132812]], "actual_norm": [[82.77973937988281, 144.49253845214844, 45.828460693359375, 29.598468780517578, 197.741943359375, 150.55831909179688, 94.59871673583984, 127.08689880371094, 173.58749389648438, 717.9981079101562, 929.3573608398438, 935.9592895507812]], "relative_norm": [[0.005233955569565296, 0.18461133539676666, 0.039668936282396317, 0.026149684563279152, 0.18595166504383087, 0.13951876759529114, 0.09132958948612213, 0.12772522866725922, 0.18060557544231415, 0.7463390827178955, 0.8726069927215576, 0.8936481475830078]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 27914, 2263, 19461, 17672, 11413, 6747, 42225, 1852, 7855], "first_top_logits": [25.625, 21.25, 20.5, 19.875, 18.375, 18.25, 17.875, 17.875, 17.875, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.4832547497935593, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.8788580894470215, "target_logp": -15.068185806274414, "log_odds": -12.189327716827393, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-1.9039748907089233, -0.9748470783233643, -3.6238969187252223e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-14.967401504516602, -0.10078433901071548]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45852661132812, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.0554656982422, 92.1599349975586, 132.1023712158203, 170.2892608642578, 715.090576171875, 931.8616333007812, 931.2835083007812]], "actual_norm": [[82.85334777832031, 147.46371459960938, 43.423404693603516, 33.40524673461914, 200.88450622558594, 145.08753967285156, 92.14484405517578, 132.184814453125, 170.33209228515625, 715.0519409179688, 931.7749633789062, 931.2001953125]], "relative_norm": [[0.00523860938847065, 0.18610942363739014, 0.03748602047562599, 0.029460705816745758, 0.18928031623363495, 0.13421086966991425, 0.08881732076406479, 0.1331888735294342, 0.17712217569351196, 0.7418689131736755, 0.875714898109436, 0.8882824182510376]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.80696487426758, 29.542028427124023, 197.76524353027344, 150.54234313964844, 94.49384307861328, 127.13677215576172, 173.5519256591797, 718.0108642578125, 929.3435668945312, 936.0150756835938]], "actual_norm": [[82.85334777832031, 144.5069580078125, 45.77998733520508, 29.591968536376953, 197.7850799560547, 150.62091064453125, 94.50958251953125, 127.14595031738281, 173.54649353027344, 718.0091552734375, 929.45654296875, 936.0003662109375]], "relative_norm": [[0.00523860938847065, 0.18462976813316345, 0.03962697833776474, 0.026143942028284073, 0.18599224090576172, 0.1395767629146576, 0.09124353528022766, 0.1277845799922943, 0.18056292831897736, 0.7463505864143372, 0.8727001547813416, 0.89368736743927]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " rupee.  The currency used in the country containing Hyderabad", "output_ids": [11122, 61498, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 68447], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.80696487426758, 29.542028427124023, 197.76524353027344, 150.54234313964844, 94.49384307861328, 127.13677215576172, 173.5519256591797, 718.0108642578125, 929.3435668945312, 936.0150756835938]], "actual_norm": [[82.85334777832031, 144.5069580078125, 45.77998733520508, 29.591968536376953, 197.7850799560547, 150.62091064453125, 94.50958251953125, 127.14595031738281, 173.54649353027344, 718.0091552734375, 929.45654296875, 936.0003662109375]], "relative_norm": [[0.00523860938847065, 0.18462976813316345, 0.03962697833776474, 0.026143942028284073, 0.18599224090576172, 0.1395767629146576, 0.09124353528022766, 0.1277845799922943, 0.18056292831897736, 0.7463505864143372, 0.8727001547813416, 0.89368736743927]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [11122, 7748, 13, 27914, 19461, 892, 6747, 11, 61283, 1532], "first_top_logits": [22.25, 20.625, 19.5, 19.5, 19.125, 17.75, 17.5, 17.5, 17.375, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.415499999653548, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-03/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-03", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Mumbai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Mumbai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.70429527759552, "target_logp": -20.70077896118164, "log_odds": -18.99648368358612, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.002174871740862727, -1.7020739316940308, -4.649054244509898e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-20.595834732055664, -0.1049446240067482]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.28924560546875, 715.0905151367188, 931.861572265625, 931.283447265625]], "actual_norm": [[82.68573760986328, 147.47776794433594, 43.47292709350586, 33.491703033447266, 200.93678283691406, 145.00927734375, 92.1556396484375, 132.0988311767578, 170.3126678466797, 715.1018676757812, 931.7741088867188, 931.3163452148438]], "relative_norm": [[0.0052280118688941, 0.18612715601921082, 0.03752877190709114, 0.029536953195929527, 0.1893295794725418, 0.13413847982883453, 0.0888277217745781, 0.13310223817825317, 0.1771019697189331, 0.7419207096099854, 0.8757140636444092, 0.888393223285675]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.76522827148438, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 718.0108032226562, 929.3434448242188, 936.0150146484375]], "actual_norm": [[82.68573760986328, 144.5130157470703, 45.83205795288086, 29.6182918548584, 197.76866149902344, 150.47698974609375, 94.53404235839844, 127.10108947753906, 173.56332397460938, 718.078125, 929.30810546875, 936.0275268554688]], "relative_norm": [[0.0052280118688941, 0.18463750183582306, 0.03967205062508583, 0.02616719901561737, 0.1859768033027649, 0.13944339752197266, 0.09126715362071991, 0.12773948907852173, 0.18058043718338013, 0.746422290802002, 0.8725607395172119, 0.8937132954597473]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian Rupee.  In the country containing Mumbai, the", "output_ids": [7748, 27914, 61498, 13, 220, 758, 279, 3146, 8482, 34712, 11, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34712, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.76522827148438, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 718.0108032226562, 929.3434448242188, 936.0150146484375]], "actual_norm": [[82.68573760986328, 144.5130157470703, 45.83205795288086, 29.6182918548584, 197.76866149902344, 150.47698974609375, 94.53404235839844, 127.10108947753906, 173.56332397460938, 718.078125, 929.30810546875, 936.0275268554688]], "relative_norm": [[0.0052280118688941, 0.18463750183582306, 0.03967205062508583, 0.02616719901561737, 0.1859768033027649, 0.13944339752197266, 0.09126715362071991, 0.12773948907852173, 0.18058043718338013, 0.746422290802002, 0.8725607395172119, 0.8937132954597473]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 6747, 47564, 104456, 4882, 11122, 2263, 42225, 27914, 78252], "first_top_logits": [27.875, 20.625, 19.5, 19.25, 19.125, 18.75, 18.75, 18.625, 18.625, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.4542519166134298, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/readout", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 7.0, "target_rank": 1289.5, "source_logit": 7.84375, "target_logit": 2.15625, "top_ids": [7748, 8330, 38352, 45418, 7065, 6364, 6747, 1111, 47238, 3001, 15973, 11413], "top_tokens": [" Indian", " Australia", "uenta", ".currency", "ican", " English", " India", "ule", " Regulation", "ica", "currency", " currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 39.0, "target_rank": 455.0, "source_logit": 6.4375, "target_logit": 3.53125, "top_ids": [15973, 96694, 11413, 53661, 3693, 3001, 27908, 42189, 5088, 6364, 18174, 4787], "top_tokens": ["currency", "onis", " currency", "cla", " American", "ica", "ingo", "etz", "urrency", " English", "oret", " North"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 101.5, "target_rank": 321.5, "source_logit": 18.75, "target_logit": 16.5, "top_ids": [11413, 35602, 12463, 3263, 22956, 26138, 10465, 8330, 7855, 90475, 29973, 7214], "top_tokens": [" currency", " currencies", " colour", "\".", "\".\n", " colours", ".\",", " Australia", " British", "(currency", " licence", " foreign"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 12.5, "target_rank": 72.5, "source_logit": 16.25, "target_logit": 14.1875, "top_ids": [11413, 65001, 7214, 7855, 37198, 2293, 7748, 13224, 3220, 74508, 4427, 3146], "top_tokens": [" currency", "\"\u2014", " foreign", " British", " Commonwealth", "\u2014", " Indian", " Britain", " money", "\u00bb.", "'.", " country"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 35.5, "target_rank": 907.5, "source_logit": 5.5625, "target_logit": 2.359375, "top_ids": [7748, 3776, 21251, 603, 32671, 28453, 3731, 5737, 12751, 19833, 4229, 24355], "top_tokens": [" Indian", " One", "onia", "one", " ______", " Currency", " Red", " Group", " Roman", " Amb", " Ge", " Medium"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 21.5, "target_rank": 122.5, "source_logit": 6.34375, "target_logit": 4.6875, "top_ids": [7179, 364, 7748, 479, 11766, 4102, 362, 11413, 547, 356, 467, 21251], "top_tokens": [" International", " '", " Indian", " G", " Standard", "\u00a0", " A", " currency", " U", " C", " W", "onia"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 79.0, "target_rank": 513.0, "source_logit": 15.9375, "target_logit": 13.875, "top_ids": [11413, 35602, 3146, 5313, 13867, 29973, 5837, 60261, 22956, 58243, 91984, 43895], "top_tokens": [" currency", " currencies", " country", " national", " county", " licence", " countries", " tariff", "\".\n", " nationality", " licences", " tariffs"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 20.5, "target_rank": 88.0, "source_logit": 16.25, "target_logit": 14.625, "top_ids": [37198, 7855, 11413, 7748, 5313, 6364, 94936, 7214, 3146, 105492, 13867, 5837], "top_tokens": [" Commonwealth", " British", " currency", " Indian", " national", " English", " denomination", " foreign", " country", "\u4e2d\u534e\u4eba\u6c11", " county", " countries"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 240.0, "target_rank": 1144.0, "source_logit": 4.25, "target_logit": 2.140625, "top_ids": [3693, 4787, 4102, 89478, 4229, 5434, 6364, 271, 279, 220, 2240, 7748], "top_tokens": [" American", " North", "\u00a0", "\u2026\n", " Ge", "\u2026\n\n", " English", "\n\n", " the", " ", " None", " Indian"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 214.5, "target_rank": 160.5, "source_logit": 4.59375, "target_logit": 5.0, "top_ids": [1852, 3693, 3639, 5938, 4102, 271, 7179, 6364, 5055, 2701, 4787, 2274], "top_tokens": [" same", " American", " United", " German", "\u00a0", "\n\n", " International", " English", " National", " following", " North", " US"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 111.5, "target_rank": 735.5, "source_logit": 16.5, "target_logit": 13.0, "top_ids": [11413, 104142, 35602, 28453, 113883, 103408, 90475, 106333, 115160, 26321, 113596, 77186], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " Currency", "\u672c\u56fd", "\u6b27\u5143", "(currency", "\u5916\u6c47", "\u5176\u4ed6\u56fd\u5bb6", "Currency", "\u56fd\u7c4d", ".Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 643.5, "target_rank": 1829.0, "source_logit": 8.375, "target_logit": 6.25, "top_ids": [104142, 101237, 112397, 106333, 116866, 113883, 101406, 109442, 110065, 105492, 11413, 103408], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u5916\u6c47", "\u76f4\u8f96\u5e02", "\u672c\u56fd", "\u8d27\u5e01", "\u8d22\u653f\u90e8", "\u6c47\u7387", "\u4e2d\u534e\u4eba\u6c11", " currency", "\u6b27\u5143"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 28.5, "target_rank": 650.0, "source_logit": 16.0, "target_logit": 6.21875, "top_ids": [11413, 104142, 11192, 28453, 17692, 112455, 7748, 80189, 19551, 42364, 7855, 103408], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " Currency", " dollar", "\u82f1\u9551", " Indian", " Dollars", " USD", " Dollar", " British", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 10.0, "target_rank": 611.5, "source_logit": 18.125, "target_logit": 6.34375, "top_ids": [104142, 11413, 7748, 28453, 43783, 17692, 7855, 35602, 42364, 6747, 11192, 3639], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Indian", " Currency", " Tamil", " dollar", " British", " currencies", " Dollar", " India", " dollars", " United"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 32.0, "target_rank": 1581.0, "source_logit": 27.5, "target_logit": 10.625, "top_ids": [104142, 11413, 28453, 11192, 35602, 112455, 17692, 101237, 80189, 103408, 101406, 26321], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Currency", " dollars", " currencies", "\u82f1\u9551", " dollar", "\u7f8e\u5143", " Dollars", "\u6b27\u5143", "\u8d27\u5e01", "Currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 29.0, "target_rank": 2202.5, "source_logit": 23.625, "target_logit": 7.125, "top_ids": [104142, 11413, 28453, 35602, 101237, 101406, 26321, 17692, 77186, 112455, 45418, 43783], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Currency", " currencies", "\u7f8e\u5143", "\u8d27\u5e01", "Currency", " dollar", ".Currency", "\u82f1\u9551", ".currency", " Tamil"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 18.5, "target_rank": 1818.0, "source_logit": 18.75, "target_logit": 8.9375, "top_ids": [279, 11122, 7748, 27914, 2598, 78252, 1964, 4882, 7855, 3881, 220, 19461], "top_tokens": [" the", " ru", " Indian", " Ru", " called", " Pound", " IN", " South", " British", " known", " ", " Euro"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 4275.0, "source_logit": 22.125, "target_logit": 5.65625, "top_ids": [7748, 11122, 27914, 6747, 43783, 4882, 33345, 47564, 104456, 2672, 42225, 19461], "top_tokens": [" Indian", " ru", " Ru", " India", " Tamil", " South", " Sri", "Indian", "\u5370\u5ea6", "ru", " indian", " Euro"]}], "source_token": 6747, "target_token": 5616, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Chennai", " is", " the"], "seconds": 1.1032144166529179}
{"run_id": "eval-04/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5263264179229736, "target_logp": -17.564599990844727, "log_odds": -17.038273572921753, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.050332631915807724, -0.47596555948257446, -2.825220326485578e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-17.494163513183594, -0.07043630629777908]}}, "score_trace": {"source": [], "target": []}, "output": " Indian rupee.  What would be the effect of a", "output_ids": [7748, 11122, 61498, 13, 220, 3555, 1035, 387, 279, 2456, 315, 264], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [], "first_top_ids": [7748, 11122, 27914, 6747, 43783, 4882, 33345, 47564, 104456, 2672], "first_top_logits": [30.125, 27.25, 22.875, 22.125, 22.0, 21.375, 20.375, 20.125, 20.0, 19.75], "source_correct": true, "target_correct": false, "seconds": 0.5070757078938186, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5263264179229736, "target_logp": -17.564599990844727, "log_odds": -17.038273572921753, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.050332631915807724, -0.47596555948257446, -2.825220326485578e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-17.494163513183594, -0.07043630629777908]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian rupee.  What would be the effect of a", "output_ids": [7748, 11122, 61498, 13, 220, 3555, 1035, 387, 279, 2456, 315, 264], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 11122, 27914, 6747, 43783, 4882, 33345, 47564, 104456, 2672], "first_top_logits": [30.125, 27.25, 22.875, 22.125, 22.0, 21.375, 20.375, 20.125, 20.0, 19.75], "source_correct": true, "target_correct": false, "seconds": 0.45371050015091896, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -33.24761962890625, "target_logp": -0.07334373891353607, "log_odds": 33.174275889992714, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-32.89699172973633, -0.35058343410491943, -4.625213477993384e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-0.023083772510290146, -0.050259966403245926]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.2892608642578, 938.2904663085938, 894.1950073242188, 947.1589965820312]], "actual_norm": [[82.7910385131836, 147.4210662841797, 43.446006774902344, 33.4132194519043, 200.92709350585938, 145.02938842773438, 92.1517333984375, 132.08673095703125, 170.29197692871094, 938.2435302734375, 894.258056640625, 947.1748046875]], "relative_norm": [[0.005234669893980026, 0.18605560064315796, 0.037505533546209335, 0.029467737302184105, 0.18932044506072998, 0.13415707647800446, 0.08882395923137665, 0.1330900341272354, 0.1770804524421692, 0.9134601354598999, 0.8300420641899109, 0.8808513879776001]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4005126953125, 943.8378295898438]], "actual_norm": [[82.7910385131836, 144.49790954589844, 45.82680130004883, 29.602994918823242, 197.79165649414062, 150.53582763671875, 94.48457336425781, 127.19356536865234, 173.55307006835938, 946.3981323242188, 885.241455078125, 943.887939453125]], "relative_norm": [[0.005234669893980026, 0.18461820483207703, 0.03966750204563141, 0.02615368366241455, 0.18599842488765717, 0.13949792087078094, 0.09121939539909363, 0.12783242762088776, 0.18056976795196533, 0.921246349811554, 0.8186430335044861, 0.8802785873413086]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Chinese yuan.  What is the currency used in the country", "output_ids": [8453, 72283, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4005126953125, 943.8378295898438]], "actual_norm": [[82.7910385131836, 144.49790954589844, 45.82680130004883, 29.602994918823242, 197.79165649414062, 150.53582763671875, 94.48457336425781, 127.19356536865234, 173.55307006835938, 946.3981323242188, 885.241455078125, 943.887939453125]], "relative_norm": [[0.005234669893980026, 0.18461820483207703, 0.03966750204563141, 0.02615368366241455, 0.18599842488765717, 0.13949792087078094, 0.09121939539909363, 0.12783242762088776, 0.18056976795196533, 0.921246349811554, 0.8186430335044861, 0.8802785873413086]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8453, 8853, 5616, 13775, 2274, 72283, 220, 431, 356, 3639], "first_top_logits": [30.0, 25.125, 24.75, 24.75, 23.625, 22.625, 22.375, 22.25, 22.125, 21.75], "source_correct": false, "target_correct": true, "seconds": 0.48009270802140236, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.174773931503296, "target_logp": -18.763168334960938, "log_odds": -15.588394403457642, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-2.698908567428589, -0.4757683277130127, -9.715084888739511e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-18.717790603637695, -0.04537707194685936]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.0554656982422, 92.1599349975586, 132.1023712158203, 170.28924560546875, 938.2904663085938, 894.1949462890625, 947.1589965820312]], "actual_norm": [[82.76513671875, 147.4741973876953, 43.41781234741211, 33.481876373291016, 200.99444580078125, 145.09686279296875, 92.20323944091797, 132.1643524169922, 170.2794647216797, 938.3179321289062, 894.1246948242188, 947.1323852539062]], "relative_norm": [[0.005233032163232565, 0.18612265586853027, 0.037481192499399185, 0.029528286308050156, 0.18938390910625458, 0.13421949744224548, 0.08887360244989395, 0.13316825032234192, 0.17706744372844696, 0.913532555103302, 0.8299182653427124, 0.8808119297027588]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.806968688964844, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4004516601562, 943.837890625]], "actual_norm": [[82.76513671875, 144.47010803222656, 45.829097747802734, 29.56659698486328, 197.77349853515625, 150.56236267089844, 94.4558334350586, 127.17008209228516, 173.4987335205078, 946.47021484375, 885.4705200195312, 943.8797607421875]], "relative_norm": [[0.005233032163232565, 0.1845826804637909, 0.03966948762536049, 0.026121526956558228, 0.18598134815692902, 0.1395225077867508, 0.09119164943695068, 0.1278088241815567, 0.18051323294639587, 0.9213165640830994, 0.8188548684120178, 0.8802709579467773]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Chennai", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 63821], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.806968688964844, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4004516601562, 943.837890625]], "actual_norm": [[82.76513671875, 144.47010803222656, 45.829097747802734, 29.56659698486328, 197.77349853515625, 150.56236267089844, 94.4558334350586, 127.17008209228516, 173.4987335205078, 946.47021484375, 885.4705200195312, 943.8797607421875]], "relative_norm": [[0.005233032163232565, 0.1845826804637909, 0.03966948762536049, 0.026121526956558228, 0.18598134815692902, 0.1395225077867508, 0.09119164943695068, 0.1278088241815567, 0.18051323294639587, 0.9213165640830994, 0.8188548684120178, 0.8802709579467773]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 7748, 6633, 11122, 12122, 1140, 4185, 198, 27914, 220], "first_top_logits": [27.0, 24.625, 20.0, 19.875, 18.875, 18.375, 18.375, 18.125, 18.0, 17.875], "source_correct": false, "target_correct": false, "seconds": 0.4067658339627087, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5528736710548401, "target_logp": -44.365028381347656, "log_odds": -43.812154710292816, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.02844870276749134, -0.5243725180625916, -5.245071224635467e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-44.278404235839844, -0.08662296086549759]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.2892608642578, 938.2904663085938, 894.1950073242188, 947.1589965820312]], "actual_norm": [[82.79634094238281, 147.4833526611328, 43.45363235473633, 33.430023193359375, 200.9598846435547, 145.03907775878906, 92.17308044433594, 132.0720672607422, 170.34939575195312, 938.26806640625, 894.0840454101562, 947.1615600585938]], "relative_norm": [[0.005235005170106888, 0.1861342042684555, 0.03751211613416672, 0.029482556506991386, 0.18935135006904602, 0.13416604697704315, 0.08884453028440475, 0.1330752670764923, 0.17714016139507294, 0.9134840369224548, 0.8298805356025696, 0.8808390498161316]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4005126953125, 943.8378295898438]], "actual_norm": [[82.79634094238281, 144.5224609375, 45.831748962402344, 29.62543487548828, 197.7605438232422, 150.51473999023438, 94.4526138305664, 127.14608764648438, 173.5244598388672, 946.5663452148438, 885.3307495117188, 943.8541259765625]], "relative_norm": [[0.005235005170106888, 0.18464957177639008, 0.03967178612947464, 0.026173509657382965, 0.18596915900707245, 0.13947837054729462, 0.09118853509426117, 0.12778471410274506, 0.18053999543190002, 0.9214101433753967, 0.8187256455421448, 0.8802470564842224]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian rupee.  What is the currency of the country", "output_ids": [7748, 11122, 61498, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696487426758, 29.54202651977539, 197.76522827148438, 150.54234313964844, 94.49383544921875, 127.13677215576172, 173.5519256591797, 946.4691772460938, 885.4005126953125, 943.8378295898438]], "actual_norm": [[82.79634094238281, 144.5224609375, 45.831748962402344, 29.62543487548828, 197.7605438232422, 150.51473999023438, 94.4526138305664, 127.14608764648438, 173.5244598388672, 946.5663452148438, 885.3307495117188, 943.8541259765625]], "relative_norm": [[0.005235005170106888, 0.18464957177639008, 0.03967178612947464, 0.026173509657382965, 0.18596915900707245, 0.13947837054729462, 0.09118853509426117, 0.12778471410274506, 0.18053999543190002, 0.9214101433753967, 0.8187256455421448, 0.8802470564842224]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 6747, 104456, 11122, 43783, 47564, 4882, 12122, 7855, 89791], "first_top_logits": [22.625, 19.0, 15.25, 14.375, 14.125, 14.0625, 13.3125, 12.875, 12.8125, 12.5], "source_correct": true, "target_correct": false, "seconds": 0.4409650000743568, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6849931478500366, "target_logp": -13.875938415527344, "log_odds": -13.190945267677307, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.04995269700884819, -0.6350041031837463, -3.635817120084539e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-13.802559852600098, -0.07337883114814758]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45849609375, 43.41843032836914, 33.43308639526367, 200.94480895996094, 145.05545043945312, 92.15992736816406, 132.1023712158203, 170.28924560546875, 938.2904052734375, 894.1949462890625, 947.1589965820312]], "actual_norm": [[82.77973937988281, 147.47142028808594, 43.46282196044922, 33.45431900024414, 201.03973388671875, 145.03663635253906, 92.27825164794922, 132.09547424316406, 170.27691650390625, 938.2040405273438, 894.1829223632812, 947.166748046875]], "relative_norm": [[0.005233955569565296, 0.18611913919448853, 0.03752004727721214, 0.02950398251414299, 0.18942658603191376, 0.1341637820005417, 0.0889459103345871, 0.13309885561466217, 0.17706479132175446, 0.9134216904640198, 0.8299723267555237, 0.8808438777923584]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.7652130126953, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 946.4691162109375, 885.4004516601562, 943.8377685546875]], "actual_norm": [[82.77973937988281, 144.49253845214844, 45.828460693359375, 29.598468780517578, 197.741943359375, 150.55831909179688, 94.59871673583984, 127.08689880371094, 173.58749389648438, 946.5794067382812, 885.3876953125, 943.9403076171875]], "relative_norm": [[0.005233955569565296, 0.18461133539676666, 0.039668936282396317, 0.026149684563279152, 0.18595166504383087, 0.13951876759529114, 0.09132958948612213, 0.12772522866725922, 0.18060557544231415, 0.9214228391647339, 0.8187782764434814, 0.8803274035453796]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian rupee.  What is the currency of the country", "output_ids": [7748, 11122, 61498, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.7652130126953, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 946.4691162109375, 885.4004516601562, 943.8377685546875]], "actual_norm": [[82.77973937988281, 144.49253845214844, 45.828460693359375, 29.598468780517578, 197.741943359375, 150.55831909179688, 94.59871673583984, 127.08689880371094, 173.58749389648438, 946.5794067382812, 885.3876953125, 943.9403076171875]], "relative_norm": [[0.005233955569565296, 0.18461133539676666, 0.039668936282396317, 0.026149684563279152, 0.18595166504383087, 0.13951876759529114, 0.09132958948612213, 0.12772522866725922, 0.18060557544231415, 0.9214228391647339, 0.8187782764434814, 0.8803274035453796]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 33345, 43783, 4882, 19461, 27914, 7855, 26392, 17672, 2263], "first_top_logits": [25.75, 21.75, 21.375, 21.0, 20.375, 19.875, 18.75, 18.75, 18.125, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.42586162500083447, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.3775324821472168, "target_logp": -12.78289794921875, "log_odds": -11.405365467071533, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.9012565016746521, -0.4762352705001831, -4.076874756719917e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-12.7252779006958, -0.05762019008398056]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45852661132812, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.0554656982422, 92.1599349975586, 132.1023712158203, 170.2892608642578, 938.29052734375, 894.1950073242188, 947.1590576171875]], "actual_norm": [[82.85334777832031, 147.46371459960938, 43.423404693603516, 33.40524673461914, 200.88450622558594, 145.08753967285156, 92.14484405517578, 132.184814453125, 170.33209228515625, 938.434814453125, 894.174072265625, 947.1184692382812]], "relative_norm": [[0.00523860938847065, 0.18610942363739014, 0.03748602047562599, 0.029460705816745758, 0.18928031623363495, 0.13421086966991425, 0.08881732076406479, 0.1331888735294342, 0.17712217569351196, 0.9136463403701782, 0.8299641013145447, 0.8807989954948425]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.80696487426758, 29.542028427124023, 197.76524353027344, 150.54234313964844, 94.49384307861328, 127.13677215576172, 173.5519256591797, 946.46923828125, 885.4005737304688, 943.8378295898438]], "actual_norm": [[82.85334777832031, 144.5069580078125, 45.77998733520508, 29.591968536376953, 197.7850799560547, 150.62091064453125, 94.50958251953125, 127.14595031738281, 173.54649353027344, 946.4030151367188, 885.3037719726562, 943.7047729492188]], "relative_norm": [[0.00523860938847065, 0.18462976813316345, 0.03962697833776474, 0.026143942028284073, 0.18599224090576172, 0.1395767629146576, 0.09124353528022766, 0.1277845799922943, 0.18056292831897736, 0.921251118183136, 0.8187006711959839, 0.8801077604293823]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian rupee.  What would be the effect of a", "output_ids": [7748, 11122, 61498, 13, 220, 3555, 1035, 387, 279, 2456, 315, 264], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50466918945312, 45.80696487426758, 29.542028427124023, 197.76524353027344, 150.54234313964844, 94.49384307861328, 127.13677215576172, 173.5519256591797, 946.46923828125, 885.4005737304688, 943.8378295898438]], "actual_norm": [[82.85334777832031, 144.5069580078125, 45.77998733520508, 29.591968536376953, 197.7850799560547, 150.62091064453125, 94.50958251953125, 127.14595031738281, 173.54649353027344, 946.4030151367188, 885.3037719726562, 943.7047729492188]], "relative_norm": [[0.00523860938847065, 0.18462976813316345, 0.03962697833776474, 0.026143942028284073, 0.18599224090576172, 0.1395767629146576, 0.09124353528022766, 0.1277845799922943, 0.18056292831897736, 0.921251118183136, 0.8187006711959839, 0.8801077604293823]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [11122, 7748, 13, 19461, 8522, 6747, 17672, 43783, 61124, 11], "first_top_logits": [20.875, 20.875, 19.0, 18.625, 17.375, 17.125, 16.875, 16.75, 16.5, 16.5], "source_correct": true, "target_correct": false, "seconds": 0.4500939999707043, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-04/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-04", "split": "eval", "country": "India", "currency": "Indian rupee", "aliases": ["rupee", "INR"], "city": "Chennai", "target_country": "China", "target_currency": "Chinese yuan", "target_aliases": ["yuan", "CNY", "renminbi", "RMB"], "prompt": "Fact: The currency used in the country containing Chennai is the", "source_continuation": " Indian rupee", "target_continuation": " Chinese yuan"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.8365733623504639, "target_logp": -17.475675582885742, "log_odds": -16.63910222053528, "token_scores": {"source": {"ids": [7748, 11122, 61498], "log_probs": [-0.00797161553055048, -0.8285553455352783, -4.637133679352701e-05]}, "target": {"ids": [8453, 72283], "log_probs": [-17.382604598999023, -0.09307153522968292]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 147.45851135253906, 43.418434143066406, 33.43308639526367, 200.94482421875, 145.05545043945312, 92.1599349975586, 132.1023712158203, 170.28924560546875, 938.2904052734375, 894.1949462890625, 947.1589965820312]], "actual_norm": [[82.68573760986328, 147.47776794433594, 43.47292709350586, 33.491703033447266, 200.93678283691406, 145.00927734375, 92.1556396484375, 132.0988311767578, 170.3126678466797, 938.3729858398438, 894.2788696289062, 947.1296997070312]], "relative_norm": [[0.0052280118688941, 0.18612715601921082, 0.03752877190709114, 0.029536953195929527, 0.1893295794725418, 0.13413847982883453, 0.0888277217745781, 0.13310223817825317, 0.1771019697189331, 0.9135861396789551, 0.8300613760948181, 0.8808094263076782]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.76522827148438, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 946.4691772460938, 885.4004516601562, 943.8377685546875]], "actual_norm": [[82.68573760986328, 144.5130157470703, 45.83205795288086, 29.6182918548584, 197.76866149902344, 150.47698974609375, 94.53404235839844, 127.10108947753906, 173.56332397460938, 946.4382934570312, 885.3265380859375, 943.8289794921875]], "relative_norm": [[0.0052280118688941, 0.18463750183582306, 0.03967205062508583, 0.02616719901561737, 0.1859768033027649, 0.13944339752197266, 0.09126715362071991, 0.12773948907852173, 0.18058043718338013, 0.9212854504585266, 0.8187217712402344, 0.8802235722541809]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indian Rupee.  What would be the effect of a", "output_ids": [7748, 27914, 61498, 13, 220, 3555, 1035, 387, 279, 2456, 315, 264], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 63821, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[82.78361511230469, 144.50465393066406, 45.80696105957031, 29.542024612426758, 197.76522827148438, 150.54234313964844, 94.49382781982422, 127.13676452636719, 173.5519256591797, 946.4691772460938, 885.4004516601562, 943.8377685546875]], "actual_norm": [[82.68573760986328, 144.5130157470703, 45.83205795288086, 29.6182918548584, 197.76866149902344, 150.47698974609375, 94.53404235839844, 127.10108947753906, 173.56332397460938, 946.4382934570312, 885.3265380859375, 943.8289794921875]], "relative_norm": [[0.0052280118688941, 0.18463750183582306, 0.03967205062508583, 0.02616719901561737, 0.1859768033027649, 0.13944339752197266, 0.09126715362071991, 0.12773948907852173, 0.18058043718338013, 0.9212854504585266, 0.8187217712402344, 0.8802235722541809]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [7748, 4882, 43783, 6747, 47564, 104456, 2263, 11122, 17692, 42225], "first_top_logits": [27.75, 22.25, 21.75, 19.25, 19.125, 19.0, 18.25, 18.125, 18.125, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.41581412497907877, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/readout", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 1514.5, "target_rank": 71.0, "source_logit": 2.328125, "target_logit": 5.71875, "top_ids": [8330, 38352, 3693, 7513, 4505, 3001, 47238, 45418, 6364, 1111, 56181, 7748], "top_tokens": [" Australia", "uenta", " American", " European", " Europe", "ica", " Regulation", ".currency", " English", "ule", " Philippine", " Indian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 133.0, "target_rank": 25.5, "source_logit": 5.5625, "target_logit": 7.53125, "top_ids": [15973, 11413, 96694, 3693, 53661, 28453, 5088, 6364, 4787, 5159, 3001, 45418], "top_tokens": ["currency", " currency", "onis", " American", "cla", " Currency", "urrency", " English", " North", " America", "ica", ".currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 389.5, "target_rank": 313.0, "source_logit": 16.125, "target_logit": 16.625, "top_ids": [11413, 35602, 8330, 3263, 6489, 36911, 10465, 5837, 24357, 22956, 13369, 7513], "top_tokens": [" currency", " currencies", " Australia", "\".", " international", ".\".", ".\",", " countries", " overseas", "\".\n", " Australian", " European"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 24.0, "target_rank": 437.5, "source_logit": 17.5, "target_logit": 13.125, "top_ids": [11413, 7214, 47412, 65001, 37198, 13224, 74508, 35602, 16675, 21130, 7855, 5837], "top_tokens": [" currency", " foreign", " foreigners", "\"\u2014", " Commonwealth", " Britain", "\u00bb.", " currencies", " nations", " parliament", " British", " countries"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 3113.5, "target_rank": 1409.5, "source_logit": 1.1875, "target_logit": 2.203125, "top_ids": [21251, 603, 3776, 55964, 56537, 5737, 21525, 29921, 28453, 3731, 2240, 11426], "top_tokens": ["onia", "one", " One", "\u2011", " _______,", " Group", " Ring", "\u2010", " Currency", " Red", " None", " Land"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 33.5, "target_rank": 198.0, "source_logit": 6.28125, "target_logit": 4.5625, "top_ids": [7179, 479, 364, 18174, 21251, 11413, 362, 64, 467, 3639, 11766, 2932], "top_tokens": [" International", " G", " '", "oret", "onia", " currency", " A", "a", " W", " United", " Standard", " She"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1437.0, "target_rank": 1947.0, "source_logit": 12.6875, "target_logit": 12.25, "top_ids": [11413, 35602, 13867, 5313, 3146, 30176, 3283, 8330, 29973, 5837, 91984, 12922], "top_tokens": [" currency", " currencies", " county", " national", " country", " counties", " city", " Australia", " licence", " countries", " licences", " island"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 58.5, "target_rank": 691.5, "source_logit": 16.125, "target_logit": 12.8125, "top_ids": [37198, 11413, 7855, 35602, 5837, 7513, 7214, 6364, 5313, 16847, 6489, 105492], "top_tokens": [" Commonwealth", " currency", " British", " currencies", " countries", " European", " foreign", " English", " national", " province", " international", "\u4e2d\u534e\u4eba\u6c11"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 1630.5, "target_rank": 373.0, "source_logit": 1.859375, "target_logit": 3.875, "top_ids": [3693, 89478, 4787, 2240, 279, 5434, 4229, 271, 111012, 220, 4102, 12022], "top_tokens": [" American", "\u2026\n", " North", " None", " the", "\u2026\n\n", " Ge", "\n\n", "\u4e16\u754c\u4e0a\u6700", " ", "\u00a0", " \u2026\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 111.0, "target_rank": 607.5, "source_logit": 5.71875, "target_logit": 3.390625, "top_ids": [1852, 3693, 4787, 5938, 271, 18174, 4102, 362, 3639, 4882, 479, 7179], "top_tokens": [" same", " American", " North", " German", "\n\n", "oret", "\u00a0", " A", " United", " South", " G", " International"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 689.0, "target_rank": 2184.0, "source_logit": 13.875, "target_logit": 11.5, "top_ids": [11413, 35602, 103408, 104142, 28453, 113883, 32488, 85466, 115160, 26321, 112659, 101237], "top_tokens": [" currency", " currencies", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " Currency", "\u672c\u56fd", " euros", " Euros", "\u5176\u4ed6\u56fd\u5bb6", "Currency", "\u591a\u4e2a\u56fd\u5bb6", "\u7f8e\u5143"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 1226.0, "target_rank": 6168.5, "source_logit": 5.53125, "target_logit": 2.328125, "top_ids": [104142, 116866, 101237, 106333, 103408, 112397, 110065, 113883, 116588, 109442, 106127, 101406], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u7f8e\u5143", "\u5916\u6c47", "\u6b27\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u6c47\u7387", "\u672c\u56fd", "\u6210\u5458\u56fd", "\u8d22\u653f\u90e8", "\u6b27\u76df", "\u8d27\u5e01"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 9.5, "target_rank": 6561.0, "source_logit": 19.625, "target_logit": 1.46875, "top_ids": [104142, 11413, 72283, 11192, 8453, 28453, 17692, 35602, 80189, 5616, 19551, 112455], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " yuan", " dollars", " Chinese", " Currency", " dollar", " currencies", " Dollars", " China", " USD", "\u82f1\u9551"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 9774.0, "source_logit": 26.625, "target_logit": 0.1396484375, "top_ids": [104142, 72283, 8453, 5616, 37047, 11413, 68884, 17692, 44923, 28453, 11192, 35602], "top_tokens": ["\u4eba\u6c11\u5e01", " yuan", " Chinese", " China", " Shanghai", " currency", " Yuan", " dollar", "Chinese", " Currency", " dollars", " currencies"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 33.5, "target_rank": 7191.5, "source_logit": 29.375, "target_logit": 7.625, "top_ids": [104142, 11413, 11192, 35602, 28453, 72283, 17692, 112455, 101237, 80189, 103408, 101406], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " dollars", " currencies", " Currency", " yuan", " dollar", "\u82f1\u9551", "\u7f8e\u5143", " Dollars", "\u6b27\u5143", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 8.0, "target_rank": 16652.0, "source_logit": 31.75, "target_logit": 2.0625, "top_ids": [104142, 72283, 8453, 37047, 106241, 35602, 11413, 5616, 101237, 17692, 11192, 101406], "top_tokens": ["\u4eba\u6c11\u5e01", " yuan", " Chinese", " Shanghai", "\u4e2d\u56fd\u7ecf\u6d4e", " currencies", " currency", " China", "\u7f8e\u5143", " dollar", " dollars", "\u8d27\u5e01"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 32.5, "target_rank": 6092.5, "source_logit": 18.875, "target_logit": 6.4375, "top_ids": [279, 2598, 72283, 3881, 8453, 431, 13775, 7855, 2274, 68884, 19180, 220], "top_tokens": [" the", " called", " yuan", " known", " Chinese", " R", " Ren", " British", " US", " Yuan", " Hong", " "]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 19.5, "target_rank": 3765.0, "source_logit": 20.5, "target_logit": 6.9375, "top_ids": [8453, 13775, 72283, 431, 8853, 68884, 2274, 356, 19180, 5672, 7855, 3639], "top_tokens": [" Chinese", " Ren", " yuan", " R", " People", " Yuan", " US", " C", " Hong", " ren", " British", " United"]}], "source_token": 5616, "target_token": 15948, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Shanghai", " is", " the"], "seconds": 1.1166042922995985}
{"run_id": "eval-05/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6355907320976257, "target_logp": -13.586854934692383, "log_odds": -12.951264202594757, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-0.6116617321968079, -0.023929011076688766]}, "target": {"ids": [34983, 1931], "log_probs": [-13.111661911010742, -0.47519323229789734]}}, "score_trace": {"source": [], "target": []}, "output": " Chinese yuan.  In the country that contains the city of", "output_ids": [8453, 72283, 13, 220, 758, 279, 3146, 429, 5610, 279, 3283, 315], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [], "first_top_ids": [8453, 13775, 431, 72283, 8853, 68884, 2274, 19180, 5672, 356], "first_top_logits": [28.75, 27.875, 27.0, 27.0, 25.375, 25.125, 23.125, 22.5, 22.5, 22.5], "source_correct": true, "target_correct": false, "seconds": 0.4608238749206066, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6355907320976257, "target_logp": -13.586854934692383, "log_odds": -12.951264202594757, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-0.6116617321968079, -0.023929011076688766]}, "target": {"ids": [34983, 1931], "log_probs": [-13.111661911010742, -0.47519323229789734]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Chinese yuan.  In the country that contains the city of", "output_ids": [8453, 72283, 13, 220, 758, 279, 3146, 429, 5610, 279, 3283, 315], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8453, 13775, 431, 72283, 8853, 68884, 2274, 19180, 5672, 356], "first_top_logits": [28.75, 27.875, 27.0, 27.0, 25.375, 25.125, 23.125, 22.5, 22.5, 22.5], "source_correct": true, "target_correct": false, "seconds": 0.48110516695305705, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -44.55378341674805, "target_logp": -5.381653785705566, "log_odds": 39.17212963104248, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-44.53145217895508, -0.022329337894916534]}, "target": {"ids": [34983, 1931], "log_probs": [-4.9064507484436035, -0.47520309686660767]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.2330322265625, 15.2520751953125, 137.0811004638672, 20.175687789916992, 254.13070678710938, 151.96029663085938, 23.830686569213867, 109.77145385742188, 205.3354949951172, 989.5822143554688, 920.2776489257812, 1301.627197265625]], "relative_norm": [[0.004946478642523289, 0.019486861303448677, 0.11865686625242233, 0.017824837937951088, 0.2389782816171646, 0.14081794023513794, 0.02300715073943138, 0.11032281070947647, 0.21363714337348938, 0.9463944435119629, 0.8334413170814514, 1.271475911140442]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.2330322265625, 15.2520751953125, 137.0811004638672, 20.175687789916992, 254.13070678710938, 151.96029663085938, 23.830686569213867, 109.77145385742188, 205.3354949951172, 989.5822143554688, 920.2776489257812, 1301.627197265625]], "relative_norm": [[0.004946478642523289, 0.019486861303448677, 0.11865686625242233, 0.017824837937951088, 0.2389782816171646, 0.14081794023513794, 0.02300715073943138, 0.11032281070947647, 0.21363714337348938, 0.9463944435119629, 0.8334413170814514, 1.271475911140442]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazil.  What is the currency used in the country containing", "output_ids": [15948, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.2330322265625, 15.2520751953125, 137.0811004638672, 20.175687789916992, 254.13070678710938, 151.96029663085938, 23.830686569213867, 109.77145385742188, 205.3354949951172, 989.5822143554688, 920.2776489257812, 1301.627197265625]], "relative_norm": [[0.004946478642523289, 0.019486861303448677, 0.11865686625242233, 0.017824837937951088, 0.2389782816171646, 0.14081794023513794, 0.02300715073943138, 0.11032281070947647, 0.21363714337348938, 0.9463944435119629, 0.8334413170814514, 1.271475911140442]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15948, 106106, 34983, 67199, 96646, 67973, 41925, 72311, 29691, 1638], "first_top_logits": [25.0, 21.25, 20.125, 16.75, 16.75, 15.25, 14.5, 14.375, 14.0625, 13.5], "source_correct": false, "target_correct": false, "seconds": 0.43413899978622794, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -10.476810455322266, "target_logp": -18.544343948364258, "log_odds": -8.067533493041992, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-10.458427429199219, -0.01838301122188568]}, "target": {"ids": [34983, 1931], "log_probs": [-18.11467742919922, -0.4296673536300659]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.09539794921875, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.26399230957031, 15.241968154907227, 137.13209533691406, 20.181869506835938, 254.05245971679688, 151.96865844726562, 23.82248878479004, 109.82122039794922, 205.31607055664062, 989.4400024414062, 920.2108154296875, 1301.7137451171875]], "relative_norm": [[0.0049484362825751305, 0.01947394758462906, 0.1187010109424591, 0.017830299213528633, 0.2389046996831894, 0.14082568883895874, 0.0229992363601923, 0.11037282645702362, 0.21361693739891052, 0.9462584257125854, 0.8333808183670044, 1.2715604305267334]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.09539794921875, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.26399230957031, 15.241968154907227, 137.13209533691406, 20.181869506835938, 254.05245971679688, 151.96865844726562, 23.82248878479004, 109.82122039794922, 205.31607055664062, 989.4400024414062, 920.2108154296875, 1301.7137451171875]], "relative_norm": [[0.0049484362825751305, 0.01947394758462906, 0.1187010109424591, 0.017830299213528633, 0.2389046996831894, 0.14082568883895874, 0.0229992363601923, 0.11037282645702362, 0.21361693739891052, 0.9462584257125854, 0.8333808183670044, 1.2715604305267334]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Shanghai", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 37047], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.09539794921875, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.26399230957031, 15.241968154907227, 137.13209533691406, 20.181869506835938, 254.05245971679688, 151.96865844726562, 23.82248878479004, 109.82122039794922, 205.31607055664062, 989.4400024414062, 920.2108154296875, 1301.7137451171875]], "relative_norm": [[0.0049484362825751305, 0.01947394758462906, 0.1187010109424591, 0.017830299213528633, 0.2389046996831894, 0.14082568883895874, 0.0229992363601923, 0.11037282645702362, 0.21361693739891052, 0.9462584257125854, 0.8333808183670044, 1.2715604305267334]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 393, 6633, 19551, 220, 72283, 1852, 13, 5313, 1887], "first_top_logits": [25.75, 21.25, 19.75, 18.375, 18.125, 18.0, 17.75, 17.75, 17.75, 17.75], "source_correct": false, "target_correct": false, "seconds": 0.41632841574028134, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.4144294261932373, "target_logp": -48.19596862792969, "log_odds": -45.78153920173645, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-2.391371011734009, -0.023058494552969933]}, "target": {"ids": [34983, 1931], "log_probs": [-47.76637268066406, -0.42959538102149963]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23072814941406, 15.245043754577637, 137.09268188476562, 20.182119369506836, 254.067626953125, 152.03741455078125, 23.83316993713379, 109.74964141845703, 205.2641143798828, 989.5296020507812, 920.2120971679688, 1301.789306640625]], "relative_norm": [[0.004946332890540361, 0.019477877765893936, 0.11866689473390579, 0.017830519005656242, 0.2389189600944519, 0.14088940620422363, 0.02300954796373844, 0.11030089110136032, 0.21356287598609924, 0.9463440775871277, 0.8333819508552551, 1.2716342210769653]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23072814941406, 15.245043754577637, 137.09268188476562, 20.182119369506836, 254.067626953125, 152.03741455078125, 23.83316993713379, 109.74964141845703, 205.2641143798828, 989.5296020507812, 920.2120971679688, 1301.789306640625]], "relative_norm": [[0.004946332890540361, 0.019477877765893936, 0.11866689473390579, 0.017830519005656242, 0.2389189600944519, 0.14088940620422363, 0.02300954796373844, 0.11030089110136032, 0.21356287598609924, 0.9463440775871277, 0.8333819508552551, 1.2716342210769653]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " China Renminbi.  In the country containing Shanghai,", "output_ids": [5616, 13775, 1065, 8221, 13, 220, 758, 279, 3146, 8482, 37047, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23072814941406, 15.245043754577637, 137.09268188476562, 20.182119369506836, 254.067626953125, 152.03741455078125, 23.83316993713379, 109.74964141845703, 205.2641143798828, 989.5296020507812, 920.2120971679688, 1301.789306640625]], "relative_norm": [[0.004946332890540361, 0.019477877765893936, 0.11866689473390579, 0.017830519005656242, 0.2389189600944519, 0.14088940620422363, 0.02300954796373844, 0.11030089110136032, 0.21356287598609924, 0.9463440775871277, 0.8333819508552551, 1.2716342210769653]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [5616, 8453, 356, 220, 279, 72858, 3220, 330, 4982, 8853], "first_top_logits": [24.875, 22.625, 20.5, 20.5, 19.25, 18.875, 18.5, 18.5, 18.25, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.4070585002191365, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.408098965883255, "target_logp": -14.112018585205078, "log_odds": -13.703919619321823, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-0.3868144750595093, -0.021284496411681175]}, "target": {"ids": [34983, 1931], "log_probs": [-13.63681411743164, -0.4752042889595032]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215645790100098, 137.09536743164062, 20.127614974975586, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.20650482177734, 15.285202026367188, 137.1277618408203, 20.159942626953125, 253.99818420410156, 151.9908447265625, 23.84870719909668, 109.8824234008789, 205.2444305419922, 989.5718383789062, 920.3439331054688, 1301.56787109375]], "relative_norm": [[0.004944801330566406, 0.019529186189174652, 0.11869725584983826, 0.017810925841331482, 0.23885366320610046, 0.14084625244140625, 0.0230245478451252, 0.11043433845043182, 0.2135424017906189, 0.9463844895362854, 0.833501398563385, 1.2714179754257202]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215645790100098, 137.09536743164062, 20.127614974975586, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.20650482177734, 15.285202026367188, 137.1277618408203, 20.159942626953125, 253.99818420410156, 151.9908447265625, 23.84870719909668, 109.8824234008789, 205.2444305419922, 989.5718383789062, 920.3439331054688, 1301.56787109375]], "relative_norm": [[0.004944801330566406, 0.019529186189174652, 0.11869725584983826, 0.017810925841331482, 0.23885366320610046, 0.14084625244140625, 0.0230245478451252, 0.11043433845043182, 0.2135424017906189, 0.9463844895362854, 0.833501398563385, 1.2714179754257202]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Chinese yuan.  The currency used in the country containing the", "output_ids": [8453, 72283, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215645790100098, 137.09536743164062, 20.127614974975586, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.20650482177734, 15.285202026367188, 137.1277618408203, 20.159942626953125, 253.99818420410156, 151.9908447265625, 23.84870719909668, 109.8824234008789, 205.2444305419922, 989.5718383789062, 920.3439331054688, 1301.56787109375]], "relative_norm": [[0.004944801330566406, 0.019529186189174652, 0.11869725584983826, 0.017810925841331482, 0.23885366320610046, 0.14084625244140625, 0.0230245478451252, 0.11043433845043182, 0.2135424017906189, 0.9463844895362854, 0.833501398563385, 1.2714179754257202]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8453, 17672, 19461, 2274, 8585, 7855, 30, 393, 3639, 14533], "first_top_logits": [22.125, 20.375, 19.375, 18.75, 18.5, 18.5, 18.125, 17.75, 17.75, 17.25], "source_correct": true, "target_correct": false, "seconds": 0.41495649982243776, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.970521450042725, "target_logp": -14.432272911071777, "log_odds": -9.461751461029053, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-4.957118988037109, -0.013402487151324749]}, "target": {"ids": [34983, 1931], "log_probs": [-13.95711898803711, -0.47515353560447693]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.0953826904297, 20.12761688232422, 254.0797119140625, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23592376708984, 15.243687629699707, 137.13912963867188, 20.075382232666016, 254.05276489257812, 152.03253173828125, 23.82508087158203, 109.84037780761719, 205.224853515625, 989.5633544921875, 920.3619384765625, 1301.5797119140625]], "relative_norm": [[0.004946661647409201, 0.01947614550590515, 0.1187070980668068, 0.017736218869686127, 0.23890498280525208, 0.1408848762512207, 0.023001737892627716, 0.11039207875728607, 0.2135220319032669, 0.946376383304596, 0.8335176706314087, 1.2714295387268066]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.0953826904297, 20.12761688232422, 254.0797119140625, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23592376708984, 15.243687629699707, 137.13912963867188, 20.075382232666016, 254.05276489257812, 152.03253173828125, 23.82508087158203, 109.84037780761719, 205.224853515625, 989.5633544921875, 920.3619384765625, 1301.5797119140625]], "relative_norm": [[0.004946661647409201, 0.01947614550590515, 0.1187070980668068, 0.017736218869686127, 0.23890498280525208, 0.1408848762512207, 0.023001737892627716, 0.11039207875728607, 0.2135220319032669, 0.946376383304596, 0.8335176706314087, 1.2714295387268066]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " which is the official currency of the country.  What is", "output_ids": [892, 374, 279, 3946, 11413, 315, 279, 3146, 13, 220, 3555, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.0953826904297, 20.12761688232422, 254.0797119140625, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692749023438, 920.2835693359375, 1301.6727294921875]], "actual_norm": [[78.23592376708984, 15.243687629699707, 137.13912963867188, 20.075382232666016, 254.05276489257812, 152.03253173828125, 23.82508087158203, 109.84037780761719, 205.224853515625, 989.5633544921875, 920.3619384765625, 1301.5797119140625]], "relative_norm": [[0.004946661647409201, 0.01947614550590515, 0.1187070980668068, 0.017736218869686127, 0.23890498280525208, 0.1408848762512207, 0.023001737892627716, 0.11039207875728607, 0.2135220319032669, 0.946376383304596, 0.8335176706314087, 1.2714295387268066]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [892, 13, 5616, 264, 11, 2176, 304, 8853, 3639, 30], "first_top_logits": [20.0, 19.125, 17.75, 17.375, 17.0, 16.5, 16.5, 16.375, 16.375, 16.25], "source_correct": false, "target_correct": false, "seconds": 0.44184825010597706, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-05/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-05", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Shanghai", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Shanghai is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.5560333728790283, "target_logp": -12.898706436157227, "log_odds": -9.342673063278198, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-3.5311737060546875, -0.024859702214598656]}, "target": {"ids": [34983, 1931], "log_probs": [-12.468673706054688, -0.4300331175327301]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.15868377685547, 15.240118026733398, 137.13101196289062, 20.1751651763916, 254.0803680419922, 152.05624389648438, 23.865571975708008, 109.7928237915039, 205.28440856933594, 989.5238647460938, 920.2227172851562, 1301.64892578125]], "relative_norm": [[0.004941777791827917, 0.019471583887934685, 0.1187000721693039, 0.01782437600195408, 0.23893094062805176, 0.14090685546398163, 0.023040829226374626, 0.1103442907333374, 0.21358399093151093, 0.9463385939598083, 0.8333916068077087, 1.271497130393982]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.15868377685547, 15.240118026733398, 137.13101196289062, 20.1751651763916, 254.0803680419922, 152.05624389648438, 23.865571975708008, 109.7928237915039, 205.28440856933594, 989.5238647460938, 920.2227172851562, 1301.64892578125]], "relative_norm": [[0.004941777791827917, 0.019471583887934685, 0.1187000721693039, 0.01782437600195408, 0.23893094062805176, 0.14090685546398163, 0.023040829226374626, 0.1103442907333374, 0.21358399093151093, 0.9463385939598083, 0.8333916068077087, 1.271497130393982]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " USD.  What is the currency used in the country containing", "output_ids": [19551, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 37047, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 989.5692138671875, 920.2835083007812, 1301.6727294921875]], "actual_norm": [[78.15868377685547, 15.240118026733398, 137.13101196289062, 20.1751651763916, 254.0803680419922, 152.05624389648438, 23.865571975708008, 109.7928237915039, 205.28440856933594, 989.5238647460938, 920.2227172851562, 1301.64892578125]], "relative_norm": [[0.004941777791827917, 0.019471583887934685, 0.1187000721693039, 0.01782437600195408, 0.23893094062805176, 0.14090685546398163, 0.023040829226374626, 0.1103442907333374, 0.21358399093151093, 0.9463385939598083, 0.8333916068077087, 1.271497130393982]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [19551, 72283, 3639, 11888, 220, 481, 8453, 431, 13775, 547], "first_top_logits": [18.875, 18.875, 18.625, 18.125, 17.75, 17.25, 17.0, 17.0, 17.0, 16.875], "source_correct": false, "target_correct": false, "seconds": 0.42457895912230015, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/readout", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 1288.5, "target_rank": 68.0, "source_logit": 3.109375, "target_logit": 6.46875, "top_ids": [8330, 38352, 15973, 11413, 45418, 49029, 6364, 31491, 3693, 4505, 28453, 31064], "top_tokens": [" Australia", "uenta", "currency", " currency", ".currency", "ctica", " English", "uesto", " American", " Europe", " Currency", " Argentina"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 78.5, "target_rank": 32.5, "source_logit": 6.90625, "target_logit": 8.125, "top_ids": [15973, 11413, 96694, 28453, 53661, 42189, 45418, 99945, 3693, 5088, 77186, 34065], "top_tokens": ["currency", " currency", "onis", " Currency", "cla", "etz", ".currency", "\u5e01", " American", "urrency", ".Currency", "_money"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 397.0, "target_rank": 267.5, "source_logit": 14.625, "target_logit": 15.5625, "top_ids": [11413, 35602, 90475, 32488, 113883, 104142, 103408, 85466, 45418, 77186, 6323, 110065], "top_tokens": [" currency", " currencies", "(currency", " euros", "\u672c\u56fd", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " Euros", ".currency", ".Currency", " Japan", "\u6c47\u7387"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 12.0, "target_rank": 458.0, "source_logit": 19.25, "target_logit": 13.5, "top_ids": [11413, 104142, 35602, 47412, 7214, 37198, 13224, 15973, 65001, 21130, 34710, 5616], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " foreigners", " foreign", " Commonwealth", " Britain", "currency", "\"\u2014", " parliament", "_currency", " China"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 1066.0, "target_rank": 314.5, "source_logit": 2.6875, "target_logit": 4.0, "top_ids": [21251, 28453, 55964, 3731, 5737, 3776, 12751, 21525, 7748, 11413, 7513, 603], "top_tokens": ["onia", " Currency", "\u2011", " Red", " Group", " One", " Roman", " Ring", " Indian", " currency", " European", "one"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 14.0, "target_rank": 154.5, "source_logit": 7.09375, "target_logit": 4.9375, "top_ids": [7179, 18174, 11766, 21251, 479, 11413, 364, 3639, 6364, 39283, 100142, 467], "top_tokens": [" International", "oret", " Standard", "onia", " G", " currency", " '", " United", " English", ".standard", "\u6807\u51c6", " W"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 641.5, "target_rank": 564.0, "source_logit": 14.6875, "target_logit": 14.875, "top_ids": [11413, 35602, 5837, 3146, 28453, 13867, 5313, 32488, 30176, 7576, 43895, 24357], "top_tokens": [" currency", " currencies", " countries", " country", " Currency", " county", " national", " euros", " counties", " prices", " tariffs", " overseas"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 32.5, "target_rank": 915.0, "source_logit": 17.625, "target_logit": 12.875, "top_ids": [11413, 105492, 37198, 5837, 35602, 6489, 7214, 5313, 114141, 3146, 94936, 6323], "top_tokens": [" currency", "\u4e2d\u534e\u4eba\u6c11", " Commonwealth", " countries", " currencies", " international", " foreign", " national", "\u81ea\u7531\u8d38\u6613", " country", " denomination", " Japan"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 1209.0, "target_rank": 186.5, "source_logit": 2.28125, "target_logit": 4.65625, "top_ids": [3693, 4787, 6364, 4229, 279, 89478, 5434, 2598, 1083, 12022, 4102, 3639], "top_tokens": [" American", " North", " English", " Ge", " the", "\u2026\n", "\u2026\n\n", " called", " also", " \u2026\n\n", "\u00a0", " United"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 41.5, "target_rank": 367.5, "source_logit": 6.46875, "target_logit": 3.84375, "top_ids": [1852, 3693, 4102, 5938, 3639, 4882, 4787, 18174, 271, 6364, 479, 5055], "top_tokens": [" same", " American", "\u00a0", " German", " United", " South", " North", "oret", "\n\n", " English", " G", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1866.0, "target_rank": 2144.5, "source_logit": 11.0, "target_logit": 10.6875, "top_ids": [104142, 103408, 11413, 35602, 113883, 28453, 115160, 106333, 32488, 113596, 90475, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", " currency", " currencies", "\u672c\u56fd", " Currency", "\u5176\u4ed6\u56fd\u5bb6", "\u5916\u6c47", " euros", "\u56fd\u7c4d", "(currency", "\u6210\u5458\u56fd"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 1042.0, "target_rank": 6696.0, "source_logit": 6.4375, "target_logit": 2.703125, "top_ids": [104142, 116866, 112397, 116588, 113883, 105492, 99599, 101237, 109442, 106333, 110065, 116103], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u6210\u5458\u56fd", "\u672c\u56fd", "\u4e2d\u534e\u4eba\u6c11", "\u56fd\u5bb6", "\u7f8e\u5143", "\u8d22\u653f\u90e8", "\u5916\u6c47", "\u6c47\u7387", "\u666e\u901a\u8bdd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 7.0, "target_rank": 2387.0, "source_logit": 21.0, "target_logit": 3.4375, "top_ids": [104142, 8453, 11413, 72283, 11192, 28453, 5616, 2598, 17692, 80189, 19551, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " Chinese", " currency", " yuan", " dollars", " Currency", " China", " called", " dollar", " Dollars", " USD", ".Currency"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 1812.0, "source_logit": 28.5, "target_logit": 3.59375, "top_ids": [8453, 104142, 5616, 72283, 11413, 44923, 68884, 56387, 58695, 28453, 105538, 26549], "top_tokens": [" Chinese", "\u4eba\u6c11\u5e01", " China", " yuan", " currency", "Chinese", " Yuan", " chinese", "\u4e2d\u56fd", " Currency", "\u4e2d\u56fd\u7684", " Beijing"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 25.0, "target_rank": 2730.0, "source_logit": 28.625, "target_logit": 9.125, "top_ids": [104142, 11413, 11192, 72283, 35602, 28453, 17692, 80189, 101237, 101406, 77186, 103408], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " dollars", " yuan", " currencies", " Currency", " dollar", " Dollars", "\u7f8e\u5143", "\u8d27\u5e01", ".Currency", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 5.0, "target_rank": 3450.0, "source_logit": 30.75, "target_logit": 5.625, "top_ids": [104142, 72283, 8453, 106241, 5616, 11413, 44923, 35602, 68884, 82771, 101237, 28453], "top_tokens": ["\u4eba\u6c11\u5e01", " yuan", " Chinese", "\u4e2d\u56fd\u7ecf\u6d4e", " China", " currency", "Chinese", " currencies", " Yuan", " Mandarin", "\u7f8e\u5143", " Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 24.0, "target_rank": 3494.5, "source_logit": 19.5, "target_logit": 7.90625, "top_ids": [279, 2598, 72283, 3881, 431, 13775, 68884, 8453, 393, 809, 220, 5672], "top_tokens": [" the", " called", " yuan", " known", " R", " Ren", " Yuan", " Chinese", " P", " Y", " ", " ren"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 26.0, "target_rank": 3644.5, "source_logit": 17.625, "target_logit": 6.71875, "top_ids": [13775, 8453, 431, 68884, 72283, 8853, 393, 5672, 356, 220, 1251, 5429], "top_tokens": [" Ren", " Chinese", " R", " Yuan", " yuan", " People", " P", " ren", " C", " ", " people", " Republic"]}], "source_token": 5616, "target_token": 15948, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Cheng", "du", " is", " the"], "seconds": 1.130570957902819}
{"run_id": "eval-06/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.952208399772644, "target_logp": -13.877577781677246, "log_odds": -11.925369381904602, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-1.9375554323196411, -0.014652970246970654]}, "target": {"ids": [34983, 1931], "log_probs": [-13.562555313110352, -0.31502267718315125]}}, "score_trace": {"source": [], "target": []}, "output": " Renminbi.  What else do we know about Cheng", "output_ids": [13775, 1065, 8221, 13, 220, 3555, 770, 653, 582, 1414, 911, 56707], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [], "first_top_ids": [13775, 8453, 431, 68884, 72283, 393, 8853, 5672, 356, 220], "first_top_logits": [28.875, 27.5, 27.25, 25.625, 25.375, 25.25, 25.25, 24.5, 21.125, 20.875], "source_correct": true, "target_correct": false, "seconds": 0.44384762505069375, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.952208399772644, "target_logp": -13.877577781677246, "log_odds": -11.925369381904602, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-1.9375554323196411, -0.014652970246970654]}, "target": {"ids": [34983, 1931], "log_probs": [-13.562555313110352, -0.31502267718315125]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Renminbi.  What else do we know about Cheng", "output_ids": [13775, 1065, 8221, 13, 220, 3555, 770, 653, 582, 1414, 911, 56707], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [13775, 8453, 431, 68884, 72283, 393, 8853, 5672, 356, 220], "first_top_logits": [28.875, 27.5, 27.25, 25.625, 25.375, 25.25, 25.25, 24.5, 21.125, 20.875], "source_correct": true, "target_correct": false, "seconds": 0.42454137466847897, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -40.54976272583008, "target_logp": -4.9610700607299805, "log_odds": 35.5886926651001, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-40.52130889892578, -0.028454262763261795]}, "target": {"ids": [34983, 1931], "log_probs": [-4.646307945251465, -0.31476226449012756]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.45648193359375, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.11181640625, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.2330322265625, 16.92243766784668, 131.68887329101562, 17.471805572509766, 255.50119018554688, 149.43531799316406, 23.682231903076172, 112.78380584716797, 199.45433044433594, 787.06640625, 644.92529296875, 797.7811279296875, 1151.8677978515625]], "relative_norm": [[0.004946478642523289, 0.02135728858411312, 0.11368274688720703, 0.015408708713948727, 0.2407420426607132, 0.13823270797729492, 0.022827021777629852, 0.11364049464464188, 0.20740532875061035, 0.7557845711708069, 0.7006281614303589, 0.7544255256652832, 1.13335120677948]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.45648193359375, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.11181640625, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.2330322265625, 16.92243766784668, 131.68887329101562, 17.471805572509766, 255.50119018554688, 149.43531799316406, 23.682231903076172, 112.78380584716797, 199.45433044433594, 787.06640625, 644.92529296875, 797.7811279296875, 1151.8677978515625]], "relative_norm": [[0.004946478642523289, 0.02135728858411312, 0.11368274688720703, 0.015408708713948727, 0.2407420426607132, 0.13823270797729492, 0.022827021777629852, 0.11364049464464188, 0.20740532875061035, 0.7557845711708069, 0.7006281614303589, 0.7544255256652832, 1.13335120677948]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazil.  What is the currency used in the country containing", "output_ids": [15948, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.9232788085938, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.2330322265625, 15.2520751953125, 137.0811004638672, 20.175687789916992, 254.13070678710938, 151.96029663085938, 23.830686569213867, 109.77145385742188, 205.3354949951172, 790.912841796875, 646.65673828125, 794.1572265625, 1147.1380615234375]], "relative_norm": [[0.004946478642523289, 0.019486861303448677, 0.11865686625242233, 0.017824837937951088, 0.2389782816171646, 0.14081794023513794, 0.02300715073943138, 0.11032281070947647, 0.21363714337348938, 0.757405161857605, 0.7014365792274475, 0.7525111436843872, 1.130060076713562]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15948, 106106, 34983, 67973, 96646, 67199, 72311, 41925, 48359, 80948], "first_top_logits": [27.625, 23.25, 23.0, 19.0, 18.875, 18.75, 18.0, 17.125, 15.125, 15.125], "source_correct": false, "target_correct": false, "seconds": 0.40419683419167995, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -10.196359634399414, "target_logp": -18.558856964111328, "log_odds": -8.362497329711914, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-10.181540489196777, -0.01481894962489605]}, "target": {"ids": [34983, 1931], "log_probs": [-18.244041442871094, -0.31481483578681946]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.4564971923828, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.1118774414062, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.26399230957031, 16.912065505981445, 131.7189483642578, 17.444194793701172, 255.42709350585938, 149.49468994140625, 23.65569496154785, 112.84233093261719, 199.49427795410156, 787.1355590820312, 644.927734375, 797.7185668945312, 1151.7607421875]], "relative_norm": [[0.0049484362825751305, 0.021344197914004326, 0.11370870471000671, 0.01538435835391283, 0.24067223072052002, 0.13828763365745544, 0.022801442071795464, 0.11369946599006653, 0.2074468731880188, 0.7558509707450867, 0.700630784034729, 0.7543663382530212, 1.1332459449768066]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.4564971923828, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.1118774414062, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.26399230957031, 16.912065505981445, 131.7189483642578, 17.444194793701172, 255.42709350585938, 149.49468994140625, 23.65569496154785, 112.84233093261719, 199.49427795410156, 787.1355590820312, 644.927734375, 797.7185668945312, 1151.7607421875]], "relative_norm": [[0.0049484362825751305, 0.021344197914004326, 0.11370870471000671, 0.01538435835391283, 0.24067223072052002, 0.13828763365745544, 0.022801442071795464, 0.11369946599006653, 0.2074468731880188, 0.7558509707450867, 0.700630784034729, 0.7543663382530212, 1.1332459449768066]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Cheng", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 56707], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.09539794921875, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.9232177734375, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.26399230957031, 15.241968154907227, 137.13209533691406, 20.181869506835938, 254.05245971679688, 151.96865844726562, 23.82248878479004, 109.82122039794922, 205.31607055664062, 790.9390258789062, 646.612548828125, 794.10498046875, 1147.2052001953125]], "relative_norm": [[0.0049484362825751305, 0.01947394758462906, 0.1187010109424591, 0.017830299213528633, 0.2389046996831894, 0.14082568883895874, 0.0229992363601923, 0.11037282645702362, 0.21361693739891052, 0.7574302554130554, 0.7013886570930481, 0.7524616718292236, 1.1301262378692627]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 393, 281, 6633, 431, 220, 1852, 13775, 72283, 198], "first_top_logits": [26.375, 23.375, 19.75, 19.625, 19.0, 18.875, 18.75, 18.625, 18.375, 18.25], "source_correct": false, "target_correct": false, "seconds": 0.410303917247802, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.103824257850647, "target_logp": -46.774723052978516, "log_odds": -45.67089879512787, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-1.0846385955810547, -0.019185686483979225]}, "target": {"ids": [34983, 1931], "log_probs": [-46.45963668823242, -0.31508609652519226]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.45648193359375, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.11181640625, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.23072814941406, 16.919071197509766, 131.73037719726562, 17.452760696411133, 255.427490234375, 149.4754180908203, 23.670555114746094, 112.79548645019531, 199.4939422607422, 787.19287109375, 645.0374755859375, 797.8092041015625, 1151.7620849609375]], "relative_norm": [[0.004946332890540361, 0.021353039890527725, 0.11371857672929764, 0.015391913242638111, 0.24067260324954987, 0.1382698118686676, 0.02281576581299305, 0.11365225911140442, 0.20744653046131134, 0.7559059858322144, 0.7007499933242798, 0.7544520497322083, 1.1332472562789917]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.45648193359375, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.11181640625, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.23072814941406, 16.919071197509766, 131.73037719726562, 17.452760696411133, 255.427490234375, 149.4754180908203, 23.670555114746094, 112.79548645019531, 199.4939422607422, 787.19287109375, 645.0374755859375, 797.8092041015625, 1151.7620849609375]], "relative_norm": [[0.004946332890540361, 0.021353039890527725, 0.11371857672929764, 0.015391913242638111, 0.24067260324954987, 0.1382698118686676, 0.02281576581299305, 0.11365225911140442, 0.20744653046131134, 0.7559059858322144, 0.7007499933242798, 0.7544520497322083, 1.1332472562789917]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " China Renminbi.  What else do we know about", "output_ids": [5616, 13775, 1065, 8221, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07969665527344, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.9232788085938, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.23072814941406, 15.245043754577637, 137.09268188476562, 20.182119369506836, 254.067626953125, 152.03741455078125, 23.83316993713379, 109.74964141845703, 205.2641143798828, 790.9701538085938, 646.607177734375, 794.1627197265625, 1147.2066650390625]], "relative_norm": [[0.004946332890540361, 0.019477877765893936, 0.11866689473390579, 0.017830519005656242, 0.2389189600944519, 0.14088940620422363, 0.02300954796373844, 0.11030089110136032, 0.21356287598609924, 0.7574600577354431, 0.7013828754425049, 0.7525163888931274, 1.1301276683807373]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [5616, 8453, 220, 356, 279, 8853, 328, 330, 4982, 5313], "first_top_logits": [26.0, 25.375, 21.875, 21.625, 21.5, 20.125, 19.375, 19.125, 18.875, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.4008457907475531, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6997130513191223, "target_logp": -12.283935546875, "log_odds": -11.584222495555878, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-0.6835802793502808, -0.016132768243551254]}, "target": {"ids": [34983, 1931], "log_probs": [-11.93358039855957, -0.3503553867340088]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.40033721923828, 255.45648193359375, 149.41360473632812, 23.638826370239258, 112.79204559326172, 199.5048828125, 787.1117553710938, 644.99658203125, 797.765869140625, 1151.7855224609375]], "actual_norm": [[78.20650482177734, 16.983421325683594, 131.7103729248047, 17.453489303588867, 255.4385223388672, 149.42733764648438, 23.739633560180664, 112.82914733886719, 199.51568603515625, 787.1453857421875, 645.0152587890625, 797.646484375, 1151.6832275390625]], "relative_norm": [[0.004944801330566406, 0.021434253081679344, 0.11370130628347397, 0.015392555855214596, 0.24068300426006317, 0.13822533190250397, 0.02288234978914261, 0.11368618160486221, 0.2074691355228424, 0.7558603882789612, 0.7007258534431458, 0.754298210144043, 1.1331696510314941]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.40033721923828, 255.45648193359375, 149.41360473632812, 23.638826370239258, 112.79204559326172, 199.5048828125, 787.1117553710938, 644.99658203125, 797.765869140625, 1151.7855224609375]], "actual_norm": [[78.20650482177734, 16.983421325683594, 131.7103729248047, 17.453489303588867, 255.4385223388672, 149.42733764648438, 23.739633560180664, 112.82914733886719, 199.51568603515625, 787.1453857421875, 645.0152587890625, 797.646484375, 1151.6832275390625]], "relative_norm": [[0.004944801330566406, 0.021434253081679344, 0.11370130628347397, 0.015392555855214596, 0.24068300426006317, 0.13822533190250397, 0.02288234978914261, 0.11368618160486221, 0.2074691355228424, 0.7558603882789612, 0.7007258534431458, 0.754298210144043, 1.1331696510314941]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Chinese yuan.  What else do we know about the country", "output_ids": [8453, 72283, 13, 220, 3555, 770, 653, 582, 1414, 911, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.215645790100098, 137.09536743164062, 20.127614974975586, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.9232177734375, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.20650482177734, 15.285202026367188, 137.1277618408203, 20.159942626953125, 253.99818420410156, 151.9908447265625, 23.84870719909668, 109.8824234008789, 205.2444305419922, 790.842041015625, 646.5759887695312, 794.1448974609375, 1147.251220703125]], "relative_norm": [[0.004944801330566406, 0.019529186189174652, 0.11869725584983826, 0.017810925841331482, 0.23885366320610046, 0.14084625244140625, 0.0230245478451252, 0.11043433845043182, 0.2135424017906189, 0.7573373317718506, 0.7013490200042725, 0.7524994611740112, 1.130171537399292]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8453, 17672, 431, 13775, 68884, 30, 72283, 19461, 393, 220], "first_top_logits": [22.125, 21.0, 20.5, 19.875, 19.375, 18.75, 18.625, 18.625, 18.375, 18.125], "source_correct": true, "target_correct": false, "seconds": 0.4120802083052695, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.2711901664733887, "target_logp": -12.577987670898438, "log_odds": -9.306797504425049, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-3.2630019187927246, -0.008188249543309212]}, "target": {"ids": [34983, 1931], "log_probs": [-12.263001441955566, -0.3149862289428711]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.88126564025879, 131.697021484375, 17.400339126586914, 255.4564971923828, 149.4136199951172, 23.638826370239258, 112.79206085205078, 199.50489807128906, 787.1118774414062, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.23592376708984, 16.873323440551758, 131.78750610351562, 17.470726013183594, 255.4973602294922, 149.4318084716797, 23.65823745727539, 112.83818817138672, 199.4966278076172, 787.0802001953125, 645.0599975585938, 797.6983032226562, 1151.7435302734375]], "relative_norm": [[0.004946661647409201, 0.021295303478837013, 0.11376789212226868, 0.01540775690227747, 0.24073843657970428, 0.13822947442531586, 0.02280389331281185, 0.11369528621435165, 0.2074493169784546, 0.755797803401947, 0.7007744908332825, 0.7543472051620483, 1.1332290172576904]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.88126564025879, 131.697021484375, 17.400339126586914, 255.4564971923828, 149.4136199951172, 23.638826370239258, 112.79206085205078, 199.50489807128906, 787.1118774414062, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.23592376708984, 16.873323440551758, 131.78750610351562, 17.470726013183594, 255.4973602294922, 149.4318084716797, 23.65823745727539, 112.83818817138672, 199.4966278076172, 787.0802001953125, 645.0599975585938, 797.6983032226562, 1151.7435302734375]], "relative_norm": [[0.004946661647409201, 0.021295303478837013, 0.11376789212226868, 0.01540775690227747, 0.24073843657970428, 0.13822947442531586, 0.02280389331281185, 0.11369528621435165, 0.2074493169784546, 0.755797803401947, 0.7007744908332825, 0.7543472051620483, 1.1332290172576904]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": ".  The currency used in the country containing Chengdu is", "output_ids": [13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.21564769744873, 137.0953826904297, 20.12761688232422, 254.0797119140625, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.92333984375, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.23592376708984, 15.243687629699707, 137.13912963867188, 20.075382232666016, 254.05276489257812, 152.03253173828125, 23.82508087158203, 109.84037780761719, 205.224853515625, 790.9739379882812, 646.5361938476562, 794.1217651367188, 1147.1845703125]], "relative_norm": [[0.004946661647409201, 0.01947614550590515, 0.1187070980668068, 0.017736218869686127, 0.23890498280525208, 0.1408848762512207, 0.023001737892627716, 0.11039207875728607, 0.2135220319032669, 0.7574636340141296, 0.7013058662414551, 0.7524775862693787, 1.1301058530807495]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [13, 892, 5616, 68884, 8453, 304, 8853, 264, 15154, 11], "first_top_logits": [19.25, 19.125, 18.0, 18.0, 17.25, 17.125, 17.0, 16.875, 16.625, 16.5], "source_correct": false, "target_correct": false, "seconds": 0.4127194592729211, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-06/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-06", "split": "eval", "country": "China", "currency": "Chinese yuan", "aliases": ["yuan", "CNY", "renminbi", "RMB"], "city": "Chengdu", "target_country": "Brazil", "target_currency": "Brazilian real", "target_aliases": ["real", "BRL"], "prompt": "Fact: The currency used in the country containing Chengdu is the", "source_continuation": " Chinese yuan", "target_continuation": " Brazilian real"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.0322251319885254, "target_logp": -12.354087829589844, "log_odds": -10.321862697601318, "token_scores": {"source": {"ids": [8453, 72283], "log_probs": [-2.0082058906555176, -0.024019325152039528]}, "target": {"ids": [34983, 1931], "log_probs": [-12.07070541381836, -0.28338196873664856]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.4564666748047, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.1117553710938, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.15868377685547, 16.950592041015625, 131.70404052734375, 17.44398307800293, 255.4330596923828, 149.35836791992188, 23.695199966430664, 112.81332397460938, 199.5759735107422, 787.0624389648438, 644.9813232421875, 797.72412109375, 1151.670166015625]], "relative_norm": [[0.004941777791827917, 0.02139282040297985, 0.11369583755731583, 0.015384172089397907, 0.2406778484582901, 0.1381615400314331, 0.022839520126581192, 0.1136702373623848, 0.20753182470798492, 0.7557807564735413, 0.7006890177726746, 0.7543715834617615, 1.1331567764282227]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 16.881263732910156, 131.697021484375, 17.400339126586914, 255.4564666748047, 149.4136199951172, 23.638826370239258, 112.79205322265625, 199.5048828125, 787.1117553710938, 644.99658203125, 797.7659301757812, 1151.7855224609375]], "actual_norm": [[78.15868377685547, 16.950592041015625, 131.70404052734375, 17.44398307800293, 255.4330596923828, 149.35836791992188, 23.695199966430664, 112.81332397460938, 199.5759735107422, 787.0624389648438, 644.9813232421875, 797.72412109375, 1151.670166015625]], "relative_norm": [[0.004941777791827917, 0.02139282040297985, 0.11369583755731583, 0.015384172089397907, 0.2406778484582901, 0.1381615400314331, 0.022839520126581192, 0.1136702373623848, 0.20753182470798492, 0.7557807564735413, 0.7006890177726746, 0.7543715834617615, 1.1331567764282227]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " yuan.  In the country containing Chengdu, the currency", "output_ids": [72283, 13, 220, 758, 279, 3146, 8482, 56707, 1054, 11, 279, 11413], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 56707, 1054, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[78.26556396484375, 15.215646743774414, 137.0953826904297, 20.12761688232422, 254.07968139648438, 152.03602600097656, 23.796062469482422, 109.80803680419922, 205.2910919189453, 790.9232788085938, 646.5723266601562, 794.132568359375, 1147.2330322265625]], "actual_norm": [[78.15868377685547, 15.240118026733398, 137.13101196289062, 20.1751651763916, 254.0803680419922, 152.05624389648438, 23.865571975708008, 109.7928237915039, 205.28440856933594, 790.9613037109375, 646.5291137695312, 794.1612548828125, 1147.34521484375]], "relative_norm": [[0.004941777791827917, 0.019471583887934685, 0.1187000721693039, 0.01782437600195408, 0.23893094062805176, 0.14090685546398163, 0.023040829226374626, 0.1103442907333374, 0.21358399093151093, 0.7574515342712402, 0.701298177242279, 0.7525150179862976, 1.130264163017273]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [72283, 13775, 8453, 431, 809, 68884, 220, 19551, 4882, 328], "first_top_logits": [20.75, 20.625, 20.25, 19.5, 19.125, 18.875, 18.75, 18.75, 18.625, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.41513237496837974, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/readout", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 152.0, "target_rank": 192.0, "source_logit": 4.4375, "target_logit": 4.1875, "top_ids": [38352, 8330, 31491, 3693, 47238, 7748, 45418, 7065, 15973, 14042, 78997, 6364], "top_tokens": ["uenta", " Australia", "uesto", " American", " Regulation", " Indian", ".currency", "ican", "currency", "razil", "ueling", " English"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 23.5, "target_rank": 40.0, "source_logit": 7.59375, "target_logit": 6.875, "top_ids": [11413, 15973, 96694, 3693, 3001, 28453, 53661, 7065, 45418, 11496, 15154, 5088], "top_tokens": [" currency", "currency", "onis", " American", "ica", " Currency", "cla", "ican", ".currency", "ika", " Spanish", "urrency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 137.0, "target_rank": 556.5, "source_logit": 16.75, "target_logit": 13.75, "top_ids": [11413, 35602, 90475, 36911, 22956, 3263, 45418, 10346, 36445, 34274, 7873, 7214], "top_tokens": [" currency", " currencies", "(currency", ".\".", "\".\n", "\".", ".currency", ".\",\n", " internationally", " Caribbean", ".[", " foreign"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 75.0, "target_rank": 44.0, "source_logit": 15.3125, "target_logit": 16.125, "top_ids": [11413, 7214, 35602, 47412, 3220, 43479, 78253, 32284, 15973, 17692, 34710, 8585], "top_tokens": [" currency", " foreign", " currencies", " foreigners", " money", " franc", " treasury", " monetary", "currency", " dollar", "_currency", " French"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 1035.5, "target_rank": 2132.5, "source_logit": 2.234375, "target_logit": 1.3984375, "top_ids": [603, 21251, 3776, 3731, 7748, 55964, 56537, 12751, 67476, 29921, 32671, 3234], "top_tokens": ["one", "onia", " One", " Red", " Indian", "\u2011", " _______,", " Roman", "'),('", "\u2010", " ______", " State"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 113.0, "target_rank": 353.5, "source_logit": 5.125, "target_logit": 3.609375, "top_ids": [11413, 7179, 364, 11766, 64, 479, 2932, 6489, 18174, 356, 362, 105469], "top_tokens": [" currency", " International", " '", " Standard", "a", " G", " She", " international", "oret", " C", " A", "\u79cd\u7c7b"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 624.0, "target_rank": 748.0, "source_logit": 14.375, "target_logit": 14.125, "top_ids": [11413, 35602, 5313, 3146, 13867, 3283, 5837, 28453, 3033, 7214, 32679, 30176], "top_tokens": [" currency", " currencies", " national", " country", " county", " city", " countries", " Currency", " government", " foreign", " citizenship", " counties"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 227.5, "target_rank": 31.5, "source_logit": 14.4375, "target_logit": 16.5, "top_ids": [11413, 35602, 43479, 7214, 6489, 105492, 94936, 7513, 5837, 8585, 17672, 15154], "top_tokens": [" currency", " currencies", " franc", " foreign", " international", "\u4e2d\u534e\u4eba\u6c11", " denomination", " European", " countries", " French", " euro", " Spanish"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 394.5, "target_rank": 401.5, "source_logit": 3.734375, "target_logit": 3.71875, "top_ids": [3693, 2598, 1083, 279, 4787, 4229, 89478, 5434, 4102, 15503, 271, 264], "top_tokens": [" American", " called", " also", " the", " North", " Ge", "\u2026\n", "\u2026\n\n", "\u00a0", " primarily", "\n\n", " a"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 243.5, "target_rank": 205.0, "source_logit": 4.5, "target_logit": 4.71875, "top_ids": [1852, 3693, 4102, 3639, 5938, 271, 4882, 4787, 6364, 2701, 7179, 5055], "top_tokens": [" same", " American", "\u00a0", " United", " German", "\n\n", " South", " North", " English", " following", " International", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 3093.0, "target_rank": 1452.5, "source_logit": 7.78125, "target_logit": 9.3125, "top_ids": [104142, 11413, 35602, 103408, 28453, 113883, 101237, 26321, 90475, 115160, 113596, 34710], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " currencies", "\u6b27\u5143", " Currency", "\u672c\u56fd", "\u7f8e\u5143", "Currency", "(currency", "\u5176\u4ed6\u56fd\u5bb6", "\u56fd\u7c4d", "_currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 2361.5, "target_rank": 520.0, "source_logit": 3.171875, "target_logit": 5.9375, "top_ids": [104142, 101237, 116866, 106333, 112397, 103408, 113883, 110065, 105492, 104773, 104328, 99599], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u56fd\u5bb6\u6807\u51c6", "\u6b27\u5143", "\u672c\u56fd", "\u6c47\u7387", "\u4e2d\u534e\u4eba\u6c11", "\u5171\u548c\u56fd", "\u6cd5\u56fd", "\u56fd\u5bb6"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 13.0, "target_rank": 137.5, "source_logit": 18.25, "target_logit": 10.0, "top_ids": [11192, 11413, 17692, 34983, 80189, 104142, 42364, 2598, 28453, 103408, 101237, 19551], "top_tokens": [" dollars", " currency", " dollar", " Brazilian", " Dollars", "\u4eba\u6c11\u5e01", " Dollar", " called", " Currency", "\u6b27\u5143", "\u7f8e\u5143", " USD"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 150.5, "source_logit": 21.5, "target_logit": 8.875, "top_ids": [34983, 42188, 17692, 11413, 15948, 104142, 106106, 11192, 3639, 42364, 28453, 112334], "top_tokens": [" Brazilian", " Portuguese", " dollar", " currency", " Brazil", "\u4eba\u6c11\u5e01", "\u5df4\u897f", " dollars", " United", " Dollar", " Currency", "\u8461\u8404\u7259"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 29.0, "target_rank": 145.0, "source_logit": 27.375, "target_logit": 18.375, "top_ids": [11192, 11413, 104142, 17692, 101237, 80189, 35602, 28453, 103408, 42364, 101406, 32488], "top_tokens": [" dollars", " currency", "\u4eba\u6c11\u5e01", " dollar", "\u7f8e\u5143", " Dollars", " currencies", " Currency", "\u6b27\u5143", " Dollar", "\u8d27\u5e01", " euros"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 17.0, "target_rank": 172.0, "source_logit": 22.375, "target_logit": 11.375, "top_ids": [104142, 101237, 34983, 11413, 17692, 103408, 106106, 35602, 101406, 28453, 11192, 80189], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", " Brazilian", " currency", " dollar", "\u6b27\u5143", "\u5df4\u897f", " currencies", "\u8d27\u5e01", " Currency", " dollars", " Dollars"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 16.0, "target_rank": 543.0, "source_logit": 19.0, "target_logit": 10.75, "top_ids": [279, 34983, 8800, 2598, 3881, 1931, 425, 2274, 547, 24043, 22218, 3639], "top_tokens": [" the", " Brazilian", " Real", " called", " known", " real", " B", " US", " U", " Bol", " cru", " United"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 190.5, "source_logit": 19.875, "target_logit": 10.5, "top_ids": [34983, 8800, 1931, 15948, 547, 425, 3639, 24043, 2274, 42188, 22218, 20842], "top_tokens": [" Brazilian", " Real", " real", " Brazil", " U", " B", " United", " Bol", " US", " Portuguese", " cru", " Cruz"]}], "source_token": 15948, "target_token": 12270, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Rec", "ife", " is", " the"], "seconds": 1.1173427919857204}
{"run_id": "eval-07/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5283262729644775, "target_logp": -8.929383277893066, "log_odds": -8.401057004928589, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.3480876684188843, -0.18023863434791565]}, "target": {"ids": [23279, 36935], "log_probs": [-8.848087310791016, -0.08129552006721497]}}, "score_trace": {"source": [], "target": []}, "output": " Brazilian real.  What is the currency used in the country", "output_ids": [34983, 1931, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [], "first_top_ids": [34983, 8800, 1931, 15948, 547, 425, 24043, 3639, 2274, 42188], "first_top_logits": [25.75, 24.5, 23.25, 19.875, 19.25, 18.5, 18.375, 18.375, 18.125, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.5245948750525713, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5283262729644775, "target_logp": -8.929383277893066, "log_odds": -8.401057004928589, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.3480876684188843, -0.18023863434791565]}, "target": {"ids": [23279, 36935], "log_probs": [-8.848087310791016, -0.08129552006721497]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazilian real.  What is the currency used in the country", "output_ids": [34983, 1931, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [34983, 8800, 1931, 15948, 547, 425, 24043, 3639, 2274, 42188], "first_top_logits": [25.75, 24.5, 23.25, 19.875, 19.25, 18.5, 18.375, 18.375, 18.125, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.41594829177483916, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -27.066373825073242, "target_logp": -1.0103087425231934, "log_odds": 26.05606508255005, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-26.923250198364258, -0.14312325417995453]}, "target": {"ids": [23279, 36935], "log_probs": [-0.9076249003410339, -0.102683886885643]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.7783203125, 94.91777038574219, 39.88218307495117, 48.4555549621582, 159.24728393554688, 83.31804656982422, 172.31288146972656, 123.3733901977539, 161.89065551757812, 135.51939392089844, 382.33892822265625, 509.6859436035156, 657.2567138671875]], "relative_norm": [[0.028185445815324783, 0.11979279667139053, 0.034428998827934265, 0.042733851820230484, 0.15004828572273254, 0.07707200944423676, 0.16609033942222595, 0.12431051582098007, 0.16834422945976257, 0.11027903854846954, 0.42599228024482727, 0.46296292543411255, 0.6620863676071167]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.7783203125, 94.91777038574219, 39.88218307495117, 48.4555549621582, 159.24728393554688, 83.31804656982422, 172.31288146972656, 123.3733901977539, 161.89065551757812, 135.51939392089844, 382.33892822265625, 509.6859436035156, 657.2567138671875]], "relative_norm": [[0.028185445815324783, 0.11979279667139053, 0.034428998827934265, 0.042733851820230484, 0.15004828572273254, 0.07707200944423676, 0.16609033942222595, 0.12431051582098007, 0.16834422945976257, 0.11027903854846954, 0.42599228024482727, 0.46296292543411255, 0.6620863676071167]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Mexico peso.  What else do we know about Recife", "output_ids": [12270, 36935, 13, 220, 3555, 770, 653, 582, 1414, 911, 4067, 1612], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 141.8513641357422, 377.83026123046875, 501.7681884765625, 655.956298828125]], "actual_norm": [[445.7783203125, 105.85414123535156, 41.765316009521484, 49.766963958740234, 164.39324951171875, 86.6158218383789, 175.58860778808594, 122.7310562133789, 162.6553955078125, 141.87518310546875, 377.79083251953125, 501.80352783203125, 655.999267578125]], "relative_norm": [[0.028185445815324783, 0.13524487614631653, 0.03615189716219902, 0.043968167155981064, 0.1545913815498352, 0.08026479184627533, 0.16952064633369446, 0.12334750592708588, 0.16923150420188904, 0.11528978496789932, 0.4210795760154724, 0.45810720324516296, 0.6570216417312622]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [12270, 23279, 109448, 36935, 3639, 393, 9369, 17692, 84147, 2274], "first_top_logits": [29.25, 28.875, 24.25, 24.25, 23.25, 21.0, 19.875, 18.375, 18.25, 18.125], "source_correct": false, "target_correct": true, "seconds": 0.4214072497561574, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.470118999481201, "target_logp": -16.234241485595703, "log_odds": -11.764122486114502, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-4.268230438232422, -0.20188863575458527]}, "target": {"ids": [23279, 36935], "log_probs": [-16.143230438232422, -0.09101039916276932]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.91331481933594, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.7618408203125, 94.83556365966797, 39.857601165771484, 48.435035705566406, 159.24490356445312, 83.23321533203125, 172.31600952148438, 123.4017105102539, 161.92330932617188, 135.50830078125, 382.3002014160156, 509.7445068359375, 657.234619140625]], "relative_norm": [[0.028184404596686363, 0.11968904733657837, 0.034407779574394226, 0.042715754359960556, 0.15004605054855347, 0.07699353247880936, 0.1660933494567871, 0.12433905154466629, 0.16837818920612335, 0.11027000844478607, 0.4259491264820099, 0.4630161225795746, 0.6620641350746155]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.91331481933594, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.7618408203125, 94.83556365966797, 39.857601165771484, 48.435035705566406, 159.24490356445312, 83.23321533203125, 172.31600952148438, 123.4017105102539, 161.92330932617188, 135.50830078125, 382.3002014160156, 509.7445068359375, 657.234619140625]], "relative_norm": [[0.028184404596686363, 0.11968904733657837, 0.034407779574394226, 0.042715754359960556, 0.15004605054855347, 0.07699353247880936, 0.1660933494567871, 0.12433905154466629, 0.16837818920612335, 0.11027000844478607, 0.4259491264820099, 0.4630161225795746, 0.6620641350746155]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Rec", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4067], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 141.8513641357422, 377.8302917480469, 501.7681884765625, 655.956298828125]], "actual_norm": [[445.7618408203125, 105.92245483398438, 41.71953201293945, 49.761573791503906, 164.31260681152344, 86.57579803466797, 175.595947265625, 122.74858856201172, 162.7542724609375, 141.91064453125, 377.8836364746094, 501.67694091796875, 655.8847045898438]], "relative_norm": [[0.028184404596686363, 0.1353321522474289, 0.036112263798713684, 0.04396340623497963, 0.15451554954051971, 0.08022770285606384, 0.16952773928642273, 0.1233651265501976, 0.16933438181877136, 0.11531860381364822, 0.4211830198764801, 0.4579916298389435, 0.6569068431854248]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 34983, 6633, 42188, 134597, 8800, 1931, 67215, 15948, 89782], "first_top_logits": [28.625, 24.375, 22.625, 20.375, 20.25, 20.0, 20.0, 19.75, 19.5, 18.875], "source_correct": false, "target_correct": false, "seconds": 0.41907504200935364, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.6161956787109375, "target_logp": -25.988903045654297, "log_odds": -24.37270736694336, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-1.4142563343048096, -0.20193928480148315]}, "target": {"ids": [23279, 36935], "log_probs": [-25.886913299560547, -0.10198913514614105]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.8116149902344, 94.9061050415039, 39.878055572509766, 48.42984390258789, 159.27369689941406, 83.27996826171875, 172.3507843017578, 123.375244140625, 161.9055633544922, 135.57974243164062, 382.2411804199219, 509.8251037597656, 657.2130737304688]], "relative_norm": [[0.028187550604343414, 0.11977807432413101, 0.03442543372511864, 0.042711175978183746, 0.15007317066192627, 0.0770367830991745, 0.16612686216831207, 0.1243123859167099, 0.16835972666740417, 0.11032814532518387, 0.42588335275650024, 0.46308931708335876, 0.6620424389839172]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38217163085938, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213439941406, 657.2479858398438]], "actual_norm": [[445.8116149902344, 94.9061050415039, 39.878055572509766, 48.42984390258789, 159.27369689941406, 83.27996826171875, 172.3507843017578, 123.375244140625, 161.9055633544922, 135.57974243164062, 382.2411804199219, 509.8251037597656, 657.2130737304688]], "relative_norm": [[0.028187550604343414, 0.11977807432413101, 0.03442543372511864, 0.042711175978183746, 0.15007317066192627, 0.0770367830991745, 0.16612686216831207, 0.1243123859167099, 0.16835972666740417, 0.11032814532518387, 0.42588335275650024, 0.46308931708335876, 0.6620424389839172]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazil Real.  What is the currency used in the country", "output_ids": [15948, 8800, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 141.8513641357422, 377.83026123046875, 501.7681884765625, 655.956298828125]], "actual_norm": [[445.8116149902344, 105.90675354003906, 41.761749267578125, 49.76840591430664, 164.2976531982422, 86.6062240600586, 175.61862182617188, 122.75505828857422, 162.69273376464844, 141.8780059814453, 377.8121032714844, 501.6784362792969, 655.9226684570312]], "relative_norm": [[0.028187550604343414, 0.13531209528446198, 0.036148808896541595, 0.04396944120526314, 0.15450148284435272, 0.0802558958530426, 0.1695496290922165, 0.12337163090705872, 0.1692703515291214, 0.11529207974672318, 0.421103298664093, 0.4579930007457733, 0.6569448709487915]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15948, 34983, 11413, 67199, 425, 1931, 96646, 8800, 73763, 41925], "first_top_logits": [26.375, 25.25, 20.375, 20.0, 19.75, 19.5, 19.375, 19.125, 18.75, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.41457020910456777, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2244822978973389, "target_logp": -11.805933952331543, "log_odds": -10.581451654434204, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.9984598159790039, -0.22602243721485138]}, "target": {"ids": [23279, 36935], "log_probs": [-11.623459815979004, -0.1824740618467331]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.80108642578125, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.3273468017578, 123.38216400146484, 161.91331481933594, 135.52146911621094, 382.2486877441406, 509.7213134765625, 657.2479858398438]], "actual_norm": [[445.24609375, 94.88031768798828, 39.9245491027832, 48.46676254272461, 159.29112243652344, 83.31693267822266, 172.3916778564453, 123.45759582519531, 161.9347381591797, 135.67108154296875, 382.26080322265625, 509.6081237792969, 657.2979125976562]], "relative_norm": [[0.028151795268058777, 0.11974553018808365, 0.034465573728084564, 0.0427437350153923, 0.1500895917415619, 0.07707097381353378, 0.166166290640831, 0.12439536303281784, 0.16839006543159485, 0.11040247231721878, 0.4259052276611328, 0.46289223432540894, 0.6621278524398804]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.80108642578125, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.3273468017578, 123.38216400146484, 161.91331481933594, 135.52146911621094, 382.2486877441406, 509.7213134765625, 657.2479858398438]], "actual_norm": [[445.24609375, 94.88031768798828, 39.9245491027832, 48.46676254272461, 159.29112243652344, 83.31693267822266, 172.3916778564453, 123.45759582519531, 161.9347381591797, 135.67108154296875, 382.26080322265625, 509.6081237792969, 657.2979125976562]], "relative_norm": [[0.028151795268058777, 0.11974553018808365, 0.034465573728084564, 0.0427437350153923, 0.1500895917415619, 0.07707097381353378, 0.166166290640831, 0.12439536303281784, 0.16839006543159485, 0.11040247231721878, 0.4259052276611328, 0.46289223432540894, 0.6621278524398804]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Real.  What else do we know about Recife?\n\n", "output_ids": [8800, 13, 220, 3555, 770, 653, 582, 1414, 911, 4067, 1612, 1939], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.80108642578125, 105.87106323242188, 41.74107360839844, 49.73102951049805, 164.29881286621094, 86.60560607910156, 175.61312866210938, 122.72553253173828, 162.69203186035156, 141.85134887695312, 377.8302307128906, 501.7681579589844, 655.9562377929688]], "actual_norm": [[445.24609375, 105.87667083740234, 41.79520797729492, 49.739952087402344, 164.3218536376953, 86.63847351074219, 175.63565063476562, 122.72567749023438, 162.69163513183594, 141.9136199951172, 377.885986328125, 501.7691345214844, 655.98046875]], "relative_norm": [[0.028151795268058777, 0.13527366518974304, 0.03617776930332184, 0.04394430294632912, 0.15452423691749573, 0.08028578013181686, 0.1695660650730133, 0.1233421042561531, 0.16926920413970947, 0.11532101780176163, 0.4211856424808502, 0.45807579159736633, 0.6570028066635132]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8800, 34983, 1931, 19461, 17672, 3639, 11413, 431, 425, 15948], "first_top_logits": [24.0, 23.625, 22.5, 20.5, 20.5, 19.25, 17.75, 17.0, 16.375, 16.375], "source_correct": true, "target_correct": false, "seconds": 0.41578825004398823, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.9896092414855957, "target_logp": -7.931919097900391, "log_odds": -6.942309856414795, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.8289688229560852, -0.1606404036283493]}, "target": {"ids": [23279, 36935], "log_probs": [-7.8289690017700195, -0.10294989496469498]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011474609375, 94.88676452636719, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.2476806640625, 172.32737731933594, 123.3821792602539, 161.91334533691406, 135.52146911621094, 382.24871826171875, 509.72137451171875, 657.2479858398438]], "actual_norm": [[445.6524658203125, 94.85235595703125, 39.88381576538086, 48.428741455078125, 159.2681427001953, 83.27955627441406, 172.3165740966797, 123.44817352294922, 161.8982391357422, 135.52967834472656, 382.2049560546875, 509.7374267578125, 657.2048950195312]], "relative_norm": [[0.02817748859524727, 0.11971024423837662, 0.034430406987667084, 0.04271020367741585, 0.15006794035434723, 0.07703640311956406, 0.1660938858985901, 0.12438586354255676, 0.16835211217403412, 0.11028740555047989, 0.4258430004119873, 0.46300968527793884, 0.6620341539382935]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011474609375, 94.88676452636719, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.2476806640625, 172.32737731933594, 123.3821792602539, 161.91334533691406, 135.52146911621094, 382.24871826171875, 509.72137451171875, 657.2479858398438]], "actual_norm": [[445.6524658203125, 94.85235595703125, 39.88381576538086, 48.428741455078125, 159.2681427001953, 83.27955627441406, 172.3165740966797, 123.44817352294922, 161.8982391357422, 135.52967834472656, 382.2049560546875, 509.7374267578125, 657.2048950195312]], "relative_norm": [[0.02817748859524727, 0.11971024423837662, 0.034430406987667084, 0.04271020367741585, 0.15006794035434723, 0.07703640311956406, 0.1660938858985901, 0.12438586354255676, 0.16835211217403412, 0.11028740555047989, 0.4258430004119873, 0.46300968527793884, 0.6620341539382935]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazilian real.  What is the currency used in the country", "output_ids": [34983, 1931, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011474609375, 105.87107849121094, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 141.8513641357422, 377.8302917480469, 501.7682189941406, 655.956298828125]], "actual_norm": [[445.6524658203125, 105.87142944335938, 41.778236389160156, 49.768272399902344, 164.3506317138672, 86.56000518798828, 175.6226806640625, 122.77967834472656, 162.73040771484375, 141.8423309326172, 377.83453369140625, 501.7653503417969, 655.9017944335938]], "relative_norm": [[0.02817748859524727, 0.1352669596672058, 0.036163080483675, 0.043969325721263885, 0.15455129742622375, 0.08021306991577148, 0.16955354809761047, 0.1233963742852211, 0.16930954158306122, 0.11526308953762054, 0.4211283028125763, 0.45807233452796936, 0.6569240093231201]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [34983, 8800, 425, 1931, 15948, 61124, 24043, 19461, 13, 106106], "first_top_logits": [21.75, 21.5, 20.125, 19.5, 18.75, 18.25, 17.5, 17.5, 16.75, 16.5], "source_correct": true, "target_correct": false, "seconds": 0.41491470905020833, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-07/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-07", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Recife", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Recife is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.16947773098945618, "target_logp": -9.303597450256348, "log_odds": -9.134119719266891, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.008666167967021465, -0.16081155836582184]}, "target": {"ids": [23279, 36935], "log_probs": [-9.258666038513184, -0.044930972158908844]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38216400146484, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213134765625, 657.2479858398438]], "actual_norm": [[445.30841064453125, 94.86660766601562, 39.883853912353516, 48.438720703125, 159.27633666992188, 83.28202819824219, 172.35195922851562, 123.44007110595703, 161.82766723632812, 135.5912322998047, 382.2629699707031, 509.8138427734375, 657.2596435546875]], "relative_norm": [[0.0281557347625494, 0.11972822993993759, 0.03443044051527977, 0.04271900653839111, 0.15007565915584564, 0.07703869044780731, 0.1661279946565628, 0.12437770515680313, 0.16827872395515442, 0.11033749580383301, 0.4259076416492462, 0.4630790948867798, 0.6620893478393555]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 94.88675689697266, 39.88070297241211, 48.407249450683594, 159.27976989746094, 83.24767303466797, 172.32736206054688, 123.38216400146484, 161.913330078125, 135.52146911621094, 382.24871826171875, 509.7213134765625, 657.2479858398438]], "actual_norm": [[445.30841064453125, 94.86660766601562, 39.883853912353516, 48.438720703125, 159.27633666992188, 83.28202819824219, 172.35195922851562, 123.44007110595703, 161.82766723632812, 135.5912322998047, 382.2629699707031, 509.8138427734375, 657.2596435546875]], "relative_norm": [[0.0281557347625494, 0.11972822993993759, 0.03443044051527977, 0.04271900653839111, 0.15007565915584564, 0.07703869044780731, 0.1661279946565628, 0.12437770515680313, 0.16827872395515442, 0.11033749580383301, 0.4259076416492462, 0.4630790948867798, 0.6620893478393555]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazilian real.  What would be a good title for this", "output_ids": [34983, 1931, 13, 220, 3555, 1035, 387, 264, 1661, 2265, 369, 419], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4067, 1612, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[445.8011169433594, 105.87106323242188, 41.74107360839844, 49.73103332519531, 164.29881286621094, 86.6056137084961, 175.61314392089844, 122.72553253173828, 162.69203186035156, 141.8513641357422, 377.8302307128906, 501.7681579589844, 655.9562377929688]], "actual_norm": [[445.30841064453125, 105.8611831665039, 41.725807189941406, 49.74291229248047, 164.2952117919922, 86.62477111816406, 175.56112670898438, 122.7746353149414, 162.70704650878906, 141.814697265625, 377.78057861328125, 501.74237060546875, 655.9833984375]], "relative_norm": [[0.0281557347625494, 0.1352538764476776, 0.03611769527196884, 0.04394691810011864, 0.15449918806552887, 0.0802730843424797, 0.1694941222667694, 0.12339130789041519, 0.16928523778915405, 0.11524063348770142, 0.4210681617259979, 0.4580513536930084, 0.6570057272911072]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [34983, 547, 15948, 4882, 3639, 42188, 2274, 8800, 106106, 24043], "first_top_logits": [25.5, 19.5, 18.625, 18.5, 18.375, 18.125, 18.0, 17.625, 17.375, 17.0], "source_correct": true, "target_correct": false, "seconds": 0.4181914171203971, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/readout", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 184.0, "target_rank": 109.5, "source_logit": 4.40625, "target_logit": 5.0, "top_ids": [38352, 3693, 3001, 8330, 6364, 7256, 31491, 7513, 45418, 8792, 3020, 7416], "top_tokens": ["uenta", " American", "ica", " Australia", " English", "ige", "uesto", " European", ".currency", "encia", "ances", " Ang"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 36.0, "target_rank": 39.5, "source_logit": 7.03125, "target_logit": 6.90625, "top_ids": [11413, 15973, 96694, 3001, 3693, 7065, 53661, 5088, 28453, 126256, 11496, 18174], "top_tokens": [" currency", "currency", "onis", "ica", " American", "ican", "cla", "urrency", " Currency", "a\u0142", "ika", "oret"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 114.5, "target_rank": 437.0, "source_logit": 17.625, "target_logit": 15.25, "top_ids": [11413, 3263, 1189, 35602, 22956, 7214, 10465, 34274, 12922, 24357, 36911, 3146], "top_tokens": [" currency", "\".", ".\"", " currencies", "\".\n", " foreign", ".\",", " Caribbean", " island", " overseas", ".\".", " country"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 235.0, "target_rank": 103.0, "source_logit": 13.0, "target_logit": 14.1875, "top_ids": [11413, 7214, 65001, 35602, 47412, 3220, 15973, 34710, 78253, 7855, 16675, 21130], "top_tokens": [" currency", " foreign", "\"\u2014", " currencies", " foreigners", " money", "currency", "_currency", " treasury", " British", " nations", " parliament"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 2470.5, "target_rank": 2153.5, "source_logit": 1.4921875, "target_logit": 1.671875, "top_ids": [21251, 603, 55964, 28453, 29921, 3776, 7748, 67476, 46804, 56537, 12751, 9127], "top_tokens": ["onia", "one", "\u2011", " Currency", "\u2010", " One", " Indian", "'),('", " predominantly", " _______,", " Roman", " Rock"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 441.0, "target_rank": 334.0, "source_logit": 3.609375, "target_logit": 4.0, "top_ids": [364, 18174, 479, 2932, 11413, 7179, 356, 11766, 362, 64, 3639, 198], "top_tokens": [" '", "oret", " G", " She", " currency", " International", " C", " Standard", " A", "a", " United", "\n"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 735.0, "target_rank": 908.0, "source_logit": 13.5, "target_logit": 13.1875, "top_ids": [11413, 35602, 5313, 13867, 3146, 3283, 28453, 30176, 29970, 12922, 7214, 7576], "top_tokens": [" currency", " currencies", " national", " county", " country", " city", " Currency", " counties", " tourist", " island", " foreign", " prices"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 582.5, "target_rank": 74.5, "source_logit": 12.25, "target_logit": 14.75, "top_ids": [11413, 37198, 35602, 7855, 7513, 7214, 94936, 6364, 6489, 5313, 43479, 15072], "top_tokens": [" currency", " Commonwealth", " currencies", " British", " European", " foreign", " denomination", " English", " international", " national", " franc", " Kingdom"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 426.0, "target_rank": 282.5, "source_logit": 3.609375, "target_logit": 4.1875, "top_ids": [89478, 3693, 5434, 279, 4787, 12022, 2240, 2598, 4229, 220, 271, 15503], "top_tokens": ["\u2026\n", " American", "\u2026\n\n", " the", " North", " \u2026\n\n", " None", " called", " Ge", " ", "\n\n", " primarily"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 507.5, "target_rank": 372.5, "source_logit": 3.25, "target_logit": 3.671875, "top_ids": [4102, 271, 3693, 479, 4787, 1852, 11413, 220, 11, 4882, 5938, 3639], "top_tokens": ["\u00a0", "\n\n", " American", " G", " North", " same", " currency", " ", ",", " South", " German", " United"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 803.0, "target_rank": 428.0, "source_logit": 13.5625, "target_logit": 14.5625, "top_ids": [11413, 35602, 28453, 32488, 26321, 104142, 103408, 11192, 85466, 15973, 77186, 34710], "top_tokens": [" currency", " currencies", " Currency", " euros", "Currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " dollars", " Euros", "currency", ".Currency", "_currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 5095.0, "target_rank": 632.0, "source_logit": 4.75, "target_logit": 9.1875, "top_ids": [104142, 116866, 106333, 101237, 11413, 101406, 103408, 110065, 35602, 109442, 26321, 28453], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u7f8e\u5143", " currency", "\u8d27\u5e01", "\u6b27\u5143", "\u6c47\u7387", " currencies", "\u8d22\u653f\u90e8", "Currency", " Currency"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 198.5, "target_rank": 81.0, "source_logit": 8.75, "target_logit": 11.5625, "top_ids": [11413, 11192, 2598, 17692, 104142, 19551, 28453, 32488, 80189, 42364, 103408, 35602], "top_tokens": [" currency", " dollars", " called", " dollar", "\u4eba\u6c11\u5e01", " USD", " Currency", " euros", " Dollars", " Dollar", "\u6b27\u5143", " currencies"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 57.0, "target_rank": 22.5, "source_logit": 12.4375, "target_logit": 15.125, "top_ids": [17692, 11413, 11192, 104142, 42364, 101237, 19551, 23279, 80189, 15154, 17672, 28453], "top_tokens": [" dollar", " currency", " dollars", "\u4eba\u6c11\u5e01", " Dollar", "\u7f8e\u5143", " USD", " Mexican", " Dollars", " Spanish", " euro", " Currency"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 221.0, "target_rank": 80.0, "source_logit": 18.875, "target_logit": 23.0, "top_ids": [11192, 11413, 104142, 17692, 80189, 101237, 35602, 28453, 103408, 42364, 32488, 101406], "top_tokens": [" dollars", " currency", "\u4eba\u6c11\u5e01", " dollar", " Dollars", "\u7f8e\u5143", " currencies", " Currency", "\u6b27\u5143", " Dollar", " euros", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 96.0, "target_rank": 40.0, "source_logit": 19.0, "target_logit": 23.25, "top_ids": [104142, 101237, 17692, 11192, 11413, 103408, 101406, 80189, 84147, 42364, 28453, 35602], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", " dollar", " dollars", " currency", "\u6b27\u5143", "\u8d27\u5e01", " Dollars", " pesos", " Dollar", " Currency", " currencies"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 389.5, "target_rank": 143.0, "source_logit": 11.625, "target_logit": 13.375, "top_ids": [279, 2598, 3881, 11192, 3639, 17692, 19551, 2274, 56856, 19461, 220, 36935], "top_tokens": [" the", " called", " known", " dollars", " United", " dollar", " USD", " US", " Pes", " Euro", " ", " peso"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 219.0, "target_rank": 103.0, "source_logit": 12.5625, "target_logit": 14.25, "top_ids": [36935, 3639, 547, 393, 23279, 4254, 1375, 2274, 8800, 34983, 31280, 17692], "top_tokens": [" peso", " United", " U", " P", " Mexican", " Col", " col", " US", " Real", " Brazilian", " Costa", " dollar"]}], "source_token": 15948, "target_token": 12270, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Salvador", " is", " the"], "seconds": 1.1313029997982085}
{"run_id": "eval-08/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.312117576599121, "target_logp": -3.1036899089813232, "log_odds": 2.208427667617798, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-5.059687614440918, -0.25242993235588074]}, "target": {"ids": [23279, 36935], "log_probs": [-3.059687614440918, -0.04400235414505005]}}, "score_trace": {"source": [], "target": []}, "output": " peso.  The currency used in the country containing Salvador is", "output_ids": [36935, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [], "first_top_ids": [36935, 3639, 547, 393, 23279, 4254, 1375, 2274, 8800, 34983], "first_top_logits": [23.875, 22.375, 22.125, 22.0, 21.5, 21.25, 21.125, 20.875, 19.625, 19.5], "source_correct": false, "target_correct": true, "seconds": 0.46148633398115635, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.312117576599121, "target_logp": -3.1036899089813232, "log_odds": 2.208427667617798, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-5.059687614440918, -0.25242993235588074]}, "target": {"ids": [23279, 36935], "log_probs": [-3.059687614440918, -0.04400235414505005]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  The currency used in the country containing Salvador is", "output_ids": [36935, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 3639, 547, 393, 23279, 4254, 1375, 2274, 8800, 34983], "first_top_logits": [23.875, 22.375, 22.125, 22.0, 21.5, 21.25, 21.125, 20.875, 19.625, 19.5], "source_correct": false, "target_correct": true, "seconds": 0.4087037919089198, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.2618175446987152, "target_logp": -15.216880798339844, "log_odds": -14.955063253641129, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-0.035803280770778656, -0.22601425647735596]}, "target": {"ids": [23279, 36935], "log_probs": [-15.160802841186523, -0.05607772618532181]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7783203125, 105.85414123535156, 41.765316009521484, 49.766963958740234, 164.39324951171875, 86.6158218383789, 175.58860778808594, 122.7310562133789, 162.6553955078125, 321.6573181152344, 221.21615600585938, 239.19943237304688]], "relative_norm": [[0.028185445815324783, 0.13524487614631653, 0.03615189716219902, 0.043968167155981064, 0.1545913815498352, 0.08026479184627533, 0.16952064633369446, 0.12334750592708588, 0.16923150420188904, 0.29096314311027527, 0.2017655074596405, 0.22792576253414154]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7783203125, 105.85414123535156, 41.765316009521484, 49.766963958740234, 164.39324951171875, 86.6158218383789, 175.58860778808594, 122.7310562133789, 162.6553955078125, 321.6573181152344, 221.21615600585938, 239.19943237304688]], "relative_norm": [[0.028185445815324783, 0.13524487614631653, 0.03615189716219902, 0.043968167155981064, 0.1545913815498352, 0.08026479184627533, 0.16952064633369446, 0.12334750592708588, 0.16923150420188904, 0.29096314311027527, 0.2017655074596405, 0.22792576253414154]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Brazilian real.  What is the currency used in the country", "output_ids": [34983, 1931, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7783203125, 105.85414123535156, 41.765316009521484, 49.766963958740234, 164.39324951171875, 86.6158218383789, 175.58860778808594, 122.7310562133789, 162.6553955078125, 321.6573181152344, 221.21615600585938, 239.19943237304688]], "relative_norm": [[0.028185445815324783, 0.13524487614631653, 0.03615189716219902, 0.043968167155981064, 0.1545913815498352, 0.08026479184627533, 0.16952064633369446, 0.12334750592708588, 0.16923150420188904, 0.29096314311027527, 0.2017655074596405, 0.22792576253414154]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [34983, 15948, 547, 425, 8800, 1931, 24043, 2274, 20771, 3639], "first_top_logits": [27.375, 23.25, 22.125, 21.75, 21.5, 21.0, 20.75, 20.125, 19.875, 19.375], "source_correct": true, "target_correct": false, "seconds": 0.4171927501447499, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.793764114379883, "target_logp": -2.460175037384033, "log_odds": 2.3335890769958496, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-4.541294574737549, -0.25246965885162354]}, "target": {"ids": [23279, 36935], "log_probs": [-2.416294574737549, -0.0438803993165493]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7618408203125, 105.92245483398438, 41.71953201293945, 49.761573791503906, 164.31260681152344, 86.57579803466797, 175.595947265625, 122.74858856201172, 162.7542724609375, 321.70172119140625, 221.2708740234375, 239.12457275390625]], "relative_norm": [[0.028184404596686363, 0.1353321522474289, 0.036112263798713684, 0.04396340623497963, 0.15451554954051971, 0.08022770285606384, 0.16952773928642273, 0.1233651265501976, 0.16933438181877136, 0.2910033166408539, 0.2018154263496399, 0.2278544306755066]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7618408203125, 105.92245483398438, 41.71953201293945, 49.761573791503906, 164.31260681152344, 86.57579803466797, 175.595947265625, 122.74858856201172, 162.7542724609375, 321.70172119140625, 221.2708740234375, 239.12457275390625]], "relative_norm": [[0.028184404596686363, 0.1353321522474289, 0.036112263798713684, 0.04396340623497963, 0.15451554954051971, 0.08022770285606384, 0.16952773928642273, 0.1233651265501976, 0.16933438181877136, 0.2910033166408539, 0.2018154263496399, 0.2278544306755066]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  The currency used in the country containing Salvador is", "output_ids": [36935, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.7618408203125, 105.92245483398438, 41.71953201293945, 49.761573791503906, 164.31260681152344, 86.57579803466797, 175.595947265625, 122.74858856201172, 162.7542724609375, 321.70172119140625, 221.2708740234375, 239.12457275390625]], "relative_norm": [[0.028184404596686363, 0.1353321522474289, 0.036112263798713684, 0.04396340623497963, 0.15451554954051971, 0.08022770285606384, 0.16952773928642273, 0.1233651265501976, 0.16933438181877136, 0.2910033166408539, 0.2018154263496399, 0.2278544306755066]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 23279, 3639, 1965, 393, 1375, 2274, 547, 17692, 4254], "first_top_logits": [23.0, 21.25, 20.875, 20.875, 20.625, 20.25, 20.125, 20.0, 19.75, 19.375], "source_correct": false, "target_correct": true, "seconds": 0.4218109999783337, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -21.623538970947266, "target_logp": -1.1047030687332153, "log_odds": 20.51883590221405, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-21.341842651367188, -0.28169673681259155]}, "target": {"ids": [23279, 36935], "log_probs": [-1.0605924129486084, -0.04411061108112335]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.8116149902344, 105.90675354003906, 41.761749267578125, 49.76840591430664, 164.2976531982422, 86.6062240600586, 175.61862182617188, 122.75505828857422, 162.69273376464844, 321.69586181640625, 221.18861389160156, 239.21438598632812]], "relative_norm": [[0.028187550604343414, 0.13531209528446198, 0.036148808896541595, 0.04396944120526314, 0.15450148284435272, 0.0802558958530426, 0.1695496290922165, 0.12337163090705872, 0.1692703515291214, 0.29099801182746887, 0.20174039900302887, 0.22794000804424286]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.8116149902344, 105.90675354003906, 41.761749267578125, 49.76840591430664, 164.2976531982422, 86.6062240600586, 175.61862182617188, 122.75505828857422, 162.69273376464844, 321.69586181640625, 221.18861389160156, 239.21438598632812]], "relative_norm": [[0.028187550604343414, 0.13531209528446198, 0.036148808896541595, 0.04396944120526314, 0.15450148284435272, 0.0802558958530426, 0.1695496290922165, 0.12337163090705872, 0.1692703515291214, 0.29099801182746887, 0.20174039900302887, 0.22794000804424286]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency used in the country containing", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.8710708618164, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.1564178466797]], "actual_norm": [[445.8116149902344, 105.90675354003906, 41.761749267578125, 49.76840591430664, 164.2976531982422, 86.6062240600586, 175.61862182617188, 122.75505828857422, 162.69273376464844, 321.69586181640625, 221.18861389160156, 239.21438598632812]], "relative_norm": [[0.028187550604343414, 0.13531209528446198, 0.036148808896541595, 0.04396944120526314, 0.15450148284435272, 0.0802558958530426, 0.1695496290922165, 0.12337163090705872, 0.1692703515291214, 0.29099801182746887, 0.20174039900302887, 0.22794000804424286]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 23279, 393, 12270, 3639, 56856, 109448, 84147, 17692, 4254], "first_top_logits": [27.125, 26.625, 25.0, 22.25, 22.0, 21.875, 20.75, 20.625, 20.125, 19.75], "source_correct": false, "target_correct": true, "seconds": 0.41650816705077887, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.127378463745117, "target_logp": -3.7245302200317383, "log_odds": 0.4028482437133789, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-3.7783474922180176, -0.34903115034103394]}, "target": {"ids": [23279, 36935], "log_probs": [-3.6533474922180176, -0.07118263095617294]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.80108642578125, 105.87106323242188, 41.74107360839844, 49.73102951049805, 164.29881286621094, 86.60560607910156, 175.61312866210938, 122.72553253173828, 162.69203186035156, 321.6934814453125, 221.18408203125, 239.15640258789062]], "actual_norm": [[445.24609375, 105.87667083740234, 41.79520797729492, 49.739952087402344, 164.3218536376953, 86.63847351074219, 175.63565063476562, 122.72567749023438, 162.69163513183594, 321.65155029296875, 221.09434509277344, 239.1814422607422]], "relative_norm": [[0.028151795268058777, 0.13527366518974304, 0.03617776930332184, 0.04394430294632912, 0.15452423691749573, 0.08028578013181686, 0.1695660650730133, 0.1233421042561531, 0.16926920413970947, 0.2909579277038574, 0.20165440440177917, 0.22790861129760742]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.80108642578125, 105.87106323242188, 41.74107360839844, 49.73102951049805, 164.29881286621094, 86.60560607910156, 175.61312866210938, 122.72553253173828, 162.69203186035156, 321.6934814453125, 221.18408203125, 239.15640258789062]], "actual_norm": [[445.24609375, 105.87667083740234, 41.79520797729492, 49.739952087402344, 164.3218536376953, 86.63847351074219, 175.63565063476562, 122.72567749023438, 162.69163513183594, 321.65155029296875, 221.09434509277344, 239.1814422607422]], "relative_norm": [[0.028151795268058777, 0.13527366518974304, 0.03617776930332184, 0.04394430294632912, 0.15452423691749573, 0.08028578013181686, 0.1695660650730133, 0.1233421042561531, 0.16926920413970947, 0.2909579277038574, 0.20165440440177917, 0.22790861129760742]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency used in the country containing", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.80108642578125, 105.87106323242188, 41.74107360839844, 49.73102951049805, 164.29881286621094, 86.60560607910156, 175.61312866210938, 122.72553253173828, 162.69203186035156, 321.6934814453125, 221.18408203125, 239.15640258789062]], "actual_norm": [[445.24609375, 105.87667083740234, 41.79520797729492, 49.739952087402344, 164.3218536376953, 86.63847351074219, 175.63565063476562, 122.72567749023438, 162.69163513183594, 321.65155029296875, 221.09434509277344, 239.1814422607422]], "relative_norm": [[0.028151795268058777, 0.13527366518974304, 0.03617776930332184, 0.04394430294632912, 0.15452423691749573, 0.08028578013181686, 0.1695660650730133, 0.1233421042561531, 0.16926920413970947, 0.2909579277038574, 0.20165440440177917, 0.22790861129760742]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 3639, 8800, 547, 2274, 19461, 4254, 1375, 23279, 34983], "first_top_logits": [22.75, 21.625, 21.5, 21.375, 20.75, 20.5, 20.375, 20.25, 20.125, 20.0], "source_correct": false, "target_correct": true, "seconds": 0.409983083140105, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.235292911529541, "target_logp": -3.5325727462768555, "log_odds": 2.7027201652526855, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-5.982875347137451, -0.2524176239967346]}, "target": {"ids": [23279, 36935], "log_probs": [-3.482875108718872, -0.049697741866111755]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011474609375, 105.87107849121094, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.15643310546875]], "actual_norm": [[445.6524658203125, 105.87142944335938, 41.778236389160156, 49.768272399902344, 164.3506317138672, 86.56000518798828, 175.6226806640625, 122.77967834472656, 162.73040771484375, 321.6593933105469, 221.14356994628906, 239.2003173828125]], "relative_norm": [[0.02817748859524727, 0.1352669596672058, 0.036163080483675, 0.043969325721263885, 0.15455129742622375, 0.08021306991577148, 0.16955354809761047, 0.1233963742852211, 0.16930954158306122, 0.2909650206565857, 0.20169930160045624, 0.2279265969991684]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011474609375, 105.87107849121094, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.15643310546875]], "actual_norm": [[445.6524658203125, 105.87142944335938, 41.778236389160156, 49.768272399902344, 164.3506317138672, 86.56000518798828, 175.6226806640625, 122.77967834472656, 162.73040771484375, 321.6593933105469, 221.14356994628906, 239.2003173828125]], "relative_norm": [[0.02817748859524727, 0.1352669596672058, 0.036163080483675, 0.043969325721263885, 0.15455129742622375, 0.08021306991577148, 0.16955354809761047, 0.1233963742852211, 0.16930954158306122, 0.2909650206565857, 0.20169930160045624, 0.2279265969991684]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency used in the country containing", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011474609375, 105.87107849121094, 41.74107360839844, 49.73103332519531, 164.298828125, 86.6056137084961, 175.61314392089844, 122.72554016113281, 162.69203186035156, 321.6935119628906, 221.18409729003906, 239.15643310546875]], "actual_norm": [[445.6524658203125, 105.87142944335938, 41.778236389160156, 49.768272399902344, 164.3506317138672, 86.56000518798828, 175.6226806640625, 122.77967834472656, 162.73040771484375, 321.6593933105469, 221.14356994628906, 239.2003173828125]], "relative_norm": [[0.02817748859524727, 0.1352669596672058, 0.036163080483675, 0.043969325721263885, 0.15455129742622375, 0.08021306991577148, 0.16955354809761047, 0.1233963742852211, 0.16930954158306122, 0.2909650206565857, 0.20169930160045624, 0.2279265969991684]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 393, 547, 3639, 23279, 56856, 2274, 4254, 8800, 18050], "first_top_logits": [23.75, 23.125, 22.125, 21.5, 21.0, 20.875, 19.75, 19.375, 19.25, 19.0], "source_correct": false, "target_correct": true, "seconds": 0.4161846246570349, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-08/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-08", "split": "eval", "country": "Brazil", "currency": "Brazilian real", "aliases": ["real", "BRL"], "city": "Salvador", "target_country": "Mexico", "target_currency": "Mexican peso", "target_aliases": ["peso", "MXN"], "prompt": "Fact: The currency used in the country containing Salvador is the", "source_continuation": " Brazilian real", "target_continuation": " Mexican peso"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.204472064971924, "target_logp": -2.509194850921631, "log_odds": 3.695277214050293, "token_scores": {"source": {"ids": [34983, 1931], "log_probs": [-5.978484630584717, -0.22598735988140106]}, "target": {"ids": [23279, 36935], "log_probs": [-2.478484869003296, -0.030709927901625633]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.87106323242188, 41.74107360839844, 49.73103332519531, 164.29881286621094, 86.6056137084961, 175.61314392089844, 122.72553253173828, 162.69203186035156, 321.6935119628906, 221.18408203125, 239.1564178466797]], "actual_norm": [[445.30841064453125, 105.8611831665039, 41.725807189941406, 49.74291229248047, 164.2952117919922, 86.62477111816406, 175.56112670898438, 122.7746353149414, 162.70704650878906, 321.66998291015625, 221.1807098388672, 239.1192626953125]], "relative_norm": [[0.0281557347625494, 0.1352538764476776, 0.03611769527196884, 0.04394691810011864, 0.15449918806552887, 0.0802730843424797, 0.1694941222667694, 0.12339130789041519, 0.16928523778915405, 0.29097461700439453, 0.20173318684101105, 0.22784936428070068]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.87106323242188, 41.74107360839844, 49.73103332519531, 164.29881286621094, 86.6056137084961, 175.61314392089844, 122.72553253173828, 162.69203186035156, 321.6935119628906, 221.18408203125, 239.1564178466797]], "actual_norm": [[445.30841064453125, 105.8611831665039, 41.725807189941406, 49.74291229248047, 164.2952117919922, 86.62477111816406, 175.56112670898438, 122.7746353149414, 162.70704650878906, 321.66998291015625, 221.1807098388672, 239.1192626953125]], "relative_norm": [[0.0281557347625494, 0.1352538764476776, 0.03611769527196884, 0.04394691810011864, 0.15449918806552887, 0.0802730843424797, 0.1694941222667694, 0.12339130789041519, 0.16928523778915405, 0.29097461700439453, 0.20173318684101105, 0.22784936428070068]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  In the country containing Salvador, the currency is", "output_ids": [36935, 13, 220, 758, 279, 3146, 8482, 48359, 11, 279, 11413, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 48359, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[445.8011169433594, 105.87106323242188, 41.74107360839844, 49.73103332519531, 164.29881286621094, 86.6056137084961, 175.61314392089844, 122.72553253173828, 162.69203186035156, 321.6935119628906, 221.18408203125, 239.1564178466797]], "actual_norm": [[445.30841064453125, 105.8611831665039, 41.725807189941406, 49.74291229248047, 164.2952117919922, 86.62477111816406, 175.56112670898438, 122.7746353149414, 162.70704650878906, 321.66998291015625, 221.1807098388672, 239.1192626953125]], "relative_norm": [[0.0281557347625494, 0.1352538764476776, 0.03611769527196884, 0.04394691810011864, 0.15449918806552887, 0.0802730843424797, 0.1694941222667694, 0.12339130789041519, 0.16928523778915405, 0.29097461700439453, 0.20173318684101105, 0.22784936428070068]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 23279, 547, 393, 3639, 4254, 1375, 2274, 17692, 56856], "first_top_logits": [24.25, 22.25, 22.25, 22.125, 21.875, 20.375, 20.125, 20.0, 19.75, 19.625], "source_correct": false, "target_correct": true, "seconds": 0.41609987523406744, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/readout", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 184.5, "target_rank": 1111.0, "source_logit": 4.59375, "target_logit": 2.59375, "top_ids": [38352, 8330, 15973, 31491, 11413, 7065, 3001, 3693, 45685, 49029, 35564, 7748], "top_tokens": ["uenta", " Australia", "currency", "uesto", " currency", "ican", "ica", " American", " Southwest", "ctica", " Southeast", " Indian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 29.0, "target_rank": 415.0, "source_logit": 7.4375, "target_logit": 4.0625, "top_ids": [11413, 15973, 96694, 3001, 53661, 7065, 28453, 3693, 27908, 42189, 11496, 21295], "top_tokens": [" currency", "currency", "onis", "ica", "cla", "ican", " Currency", " American", "ingo", "etz", "ika", "ollar"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 205.0, "target_rank": 324.5, "source_logit": 16.875, "target_logit": 15.9375, "top_ids": [11413, 35602, 6489, 36445, 24357, 7214, 34274, 8330, 90475, 5837, 85466, 23968], "top_tokens": [" currency", " currencies", " international", " internationally", " overseas", " foreign", " Caribbean", " Australia", "(currency", " countries", " Euros", " Indonesia"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 21.5, "target_rank": 172.0, "source_logit": 16.875, "target_logit": 14.0625, "top_ids": [11413, 7214, 16675, 47412, 5313, 35602, 17692, 15973, 3220, 65001, 78253, 87598], "top_tokens": [" currency", " foreign", " nations", " foreigners", " national", " currencies", " dollar", "currency", " money", "\"\u2014", " treasury", " canadian"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 2039.0, "target_rank": 4345.5, "source_logit": 1.4921875, "target_logit": 0.48828125, "top_ids": [21251, 28453, 3776, 3731, 7748, 603, 5737, 56537, 4229, 55964, 12751, 19833], "top_tokens": ["onia", " Currency", " One", " Red", " Indian", "one", " Group", " _______,", " Ge", "\u2011", " Roman", " Amb"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 230.5, "target_rank": 857.0, "source_logit": 4.125, "target_logit": 2.421875, "top_ids": [479, 11413, 364, 11766, 7179, 18174, 2932, 356, 64, 4716, 547, 28453], "top_tokens": [" G", " currency", " '", " Standard", " International", "oret", " She", " C", "a", " Bar", " U", " Currency"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 647.5, "target_rank": 4106.0, "source_logit": 13.875, "target_logit": 11.125, "top_ids": [11413, 35602, 13867, 3146, 5313, 3283, 30176, 7214, 5837, 28453, 11192, 9867], "top_tokens": [" currency", " currencies", " county", " country", " national", " city", " counties", " foreign", " countries", " Currency", " dollars", " native"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 29.0, "target_rank": 470.5, "source_logit": 16.0, "target_logit": 12.8125, "top_ids": [11413, 7214, 94936, 37198, 35602, 43479, 5313, 105492, 5837, 7513, 6489, 16847], "top_tokens": [" currency", " foreign", " denomination", " Commonwealth", " currencies", " franc", " national", "\u4e2d\u534e\u4eba\u6c11", " countries", " European", " international", " province"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 188.5, "target_rank": 7146.5, "source_logit": 4.59375, "target_logit": -0.69921875, "top_ids": [3693, 89478, 4787, 279, 4102, 5434, 4229, 6364, 2240, 1083, 12022, 271], "top_tokens": [" American", "\u2026\n", " North", " the", "\u00a0", "\u2026\n\n", " Ge", " English", " None", " also", " \u2026\n\n", "\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 116.5, "target_rank": 1536.0, "source_logit": 5.625, "target_logit": 1.9609375, "top_ids": [1852, 4102, 3693, 5938, 3639, 271, 4787, 4882, 5055, 479, 11888, 2274], "top_tokens": [" same", "\u00a0", " American", " German", " United", "\n\n", " North", " South", " National", " G", " Canadian", " US"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 426.0, "target_rank": 1155.5, "source_logit": 13.6875, "target_logit": 11.75, "top_ids": [11413, 35602, 104142, 28453, 113883, 103408, 90475, 101237, 26321, 106333, 113596, 34710], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u672c\u56fd", "\u6b27\u5143", "(currency", "\u7f8e\u5143", "Currency", "\u5916\u6c47", "\u56fd\u7c4d", "_currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 276.0, "target_rank": 3572.5, "source_logit": 7.6875, "target_logit": 2.875, "top_ids": [104142, 116866, 112397, 101237, 106333, 109442, 113883, 110065, 103408, 99599, 101406, 104773], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u7f8e\u5143", "\u5916\u6c47", "\u8d22\u653f\u90e8", "\u672c\u56fd", "\u6c47\u7387", "\u6b27\u5143", "\u56fd\u5bb6", "\u8d27\u5e01", "\u5171\u548c\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 6.0, "target_rank": 1406.0, "source_logit": 20.75, "target_logit": 4.40625, "top_ids": [11413, 11192, 23279, 17692, 19551, 12270, 80189, 42364, 101237, 28453, 104142, 109448], "top_tokens": [" currency", " dollars", " Mexican", " dollar", " USD", " Mexico", " Dollars", " Dollar", "\u7f8e\u5143", " Currency", "\u4eba\u6c11\u5e01", "\u58a8\u897f\u54e5"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 853.5, "source_logit": 22.125, "target_logit": 4.96875, "top_ids": [23279, 11413, 12270, 17692, 19551, 15154, 11192, 109448, 104142, 101237, 3639, 42364], "top_tokens": [" Mexican", " currency", " Mexico", " dollar", " USD", " Spanish", " dollars", "\u58a8\u897f\u54e5", "\u4eba\u6c11\u5e01", "\u7f8e\u5143", " United", " Dollar"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 17.0, "target_rank": 1088.5, "source_logit": 31.375, "target_logit": 11.6875, "top_ids": [11192, 11413, 101237, 17692, 80189, 104142, 28453, 19551, 35602, 42364, 101406, 23279], "top_tokens": [" dollars", " currency", "\u7f8e\u5143", " dollar", " Dollars", "\u4eba\u6c11\u5e01", " Currency", " USD", " currencies", " Dollar", "\u8d27\u5e01", " Mexican"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 10.0, "target_rank": 1546.5, "source_logit": 26.0, "target_logit": 6.25, "top_ids": [101237, 104142, 23279, 109448, 17692, 101406, 11192, 11413, 84147, 12270, 80189, 28453], "top_tokens": ["\u7f8e\u5143", "\u4eba\u6c11\u5e01", " Mexican", "\u58a8\u897f\u54e5", " dollar", "\u8d27\u5e01", " dollars", " currency", " pesos", " Mexico", " Dollars", " Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 7.5, "target_rank": 24045.0, "source_logit": 21.625, "target_logit": 1.484375, "top_ids": [279, 23279, 3639, 36935, 3881, 2274, 2598, 12270, 393, 17692, 547, 84147], "top_tokens": [" the", " Mexican", " United", " peso", " known", " US", " called", " Mexico", " P", " dollar", " U", " pesos"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 7.0, "target_rank": 54681.0, "source_logit": 20.25, "target_logit": -2.703125, "top_ids": [36935, 393, 23279, 3639, 4787, 2274, 12270, 547, 17692, 84147, 109448, 56856], "top_tokens": [" peso", " P", " Mexican", " United", " North", " US", " Mexico", " U", " dollar", " pesos", "\u58a8\u897f\u54e5", " Pes"]}], "source_token": 12270, "target_token": 8359, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Monter", "rey", " is", " the"], "seconds": 1.1528492919169366}
{"run_id": "eval-09/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.481288194656372, "target_logp": -23.243728637695312, "log_odds": -20.76244044303894, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-2.3379993438720703, -0.1432887464761734]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-22.706350326538086, -0.5370932221412659, -0.0002851079625543207]}}, "score_trace": {"source": [], "target": []}, "output": " peso.  The currency used in the country containing Guadal", "output_ids": [36935, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [], "first_top_ids": [36935, 393, 23279, 3639, 4787, 2274, 12270, 547, 17692, 84147], "first_top_logits": [27.125, 26.75, 25.5, 23.875, 21.625, 21.5, 20.25, 20.0, 19.875, 19.25], "source_correct": true, "target_correct": false, "seconds": 0.4624949162825942, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.481288194656372, "target_logp": -23.243728637695312, "log_odds": -20.76244044303894, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-2.3379993438720703, -0.1432887464761734]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-22.706350326538086, -0.5370932221412659, -0.0002851079625543207]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  The currency used in the country containing Guadal", "output_ids": [36935, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 393, 23279, 3639, 4787, 2274, 12270, 547, 17692, 84147], "first_top_logits": [27.125, 26.75, 25.5, 23.875, 21.625, 21.5, 20.25, 20.0, 19.875, 19.25], "source_correct": true, "target_correct": false, "seconds": 0.4831329579465091, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -40.08100128173828, "target_logp": -4.405174255371094, "log_odds": 35.67582702636719, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-39.76677322387695, -0.3142290413379669]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-4.1443095207214355, -0.2604586184024811, -0.000406301929615438]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67239379882812, 782.2499389648438, 739.340576171875, 965.5000610351562, 1015.1803588867188]], "actual_norm": [[398.5150451660156, 37.49421691894531, 54.68024444580078, 103.8343276977539, 212.8571319580078, 87.22295379638672, 117.56754302978516, 15.33811092376709, 224.7115478515625, 782.3276977539062, 739.4004516601562, 965.5310668945312, 1015.2986450195312]], "relative_norm": [[0.0251971073448658, 0.047320298850536346, 0.04720368608832359, 0.09157341718673706, 0.20056134462356567, 0.08068417757749557, 0.113321952521801, 0.015454616397619247, 0.23366940021514893, 0.6893224120140076, 0.8388192057609558, 0.8841490149497986, 1.0082627534866333]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 782.5341186523438, 742.5804443359375, 946.0936889648438, 1015.026123046875]], "actual_norm": [[398.5150451660156, 38.854148864746094, 55.21453094482422, 102.5206527709961, 215.6384735107422, 89.64886474609375, 119.22421264648438, 14.077310562133789, 225.23284912109375, 782.557861328125, 742.60205078125, 946.1141967773438, 1015.0018310546875]], "relative_norm": [[0.0251971073448658, 0.04891373962163925, 0.04777214303612709, 0.09030702710151672, 0.20303620398044586, 0.08292198926210403, 0.11482074111700058, 0.014191106893122196, 0.23482085764408112, 0.6911999583244324, 0.841270923614502, 0.8693789839744568, 1.0029029846191406]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russia ruble.  What would be the best way to", "output_ids": [8359, 10273, 273, 13, 220, 3555, 1035, 387, 279, 1850, 1616, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 47.4112663269043, 60.8482780456543, 99.16316223144531, 218.51290893554688, 91.53727722167969, 119.7630615234375, 16.603445053100586, 229.5046844482422, 786.9169921875, 737.2642822265625, 953.1492919921875, 1015.8197021484375]], "actual_norm": [[398.5150451660156, 47.412391662597656, 60.83124923706055, 99.16502380371094, 218.49855041503906, 91.50643157958984, 119.74905395507812, 16.58911895751953, 229.4862823486328, 786.8566284179688, 737.2658081054688, 953.104736328125, 1015.76318359375]], "relative_norm": [[0.0251971073448658, 0.06057658791542053, 0.05265529453754425, 0.08761041611433029, 0.20547068119049072, 0.08479680120944977, 0.11561078578233719, 0.016672441735863686, 0.23876434564590454, 0.6925561428070068, 0.8357970118522644, 0.8768673539161682, 1.0064477920532227]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8359, 8522, 435, 431, 3146, 10273, 104084, 4741, 3070, 3639], "first_top_logits": [30.125, 25.875, 24.0, 22.0, 19.25, 19.125, 18.625, 18.625, 18.5, 18.375], "source_correct": false, "target_correct": true, "seconds": 0.41659179143607616, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -7.162604331970215, "target_logp": -19.070266723632812, "log_odds": -11.907662391662598, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-7.001821517944336, -0.16078291833400726]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-18.75218963623047, -0.317238450050354, -0.0008400725200772285]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67237854003906, 782.2499389648438, 739.340576171875, 965.5000610351562, 1015.1802368164062]], "actual_norm": [[398.5599365234375, 37.49838638305664, 54.59830093383789, 103.8289794921875, 212.8018035888672, 87.17161560058594, 117.55985260009766, 15.413719177246094, 224.66209411621094, 782.3052978515625, 739.3866577148438, 965.5278930664062, 1015.18310546875]], "relative_norm": [[0.025199946016073227, 0.047325558960437775, 0.0471329465508461, 0.09156870096921921, 0.2005092054605484, 0.08063668757677078, 0.11331453919410706, 0.015530799515545368, 0.23361797630786896, 0.6893026828765869, 0.8388035893440247, 0.8841460943222046, 1.008147954940796]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993583679199219, 225.21766662597656, 782.5341186523438, 742.5804443359375, 946.0936889648438, 1015.026123046875]], "actual_norm": [[398.5599365234375, 38.83179473876953, 55.18263626098633, 102.50543212890625, 215.60459899902344, 89.62640380859375, 119.2352294921875, 14.005796432495117, 225.18820190429688, 782.5450439453125, 742.59033203125, 946.097900390625, 1015.0194091796875]], "relative_norm": [[0.025199946016073227, 0.04888559505343437, 0.047744546085596085, 0.09029361605644226, 0.20300430059432983, 0.08290121704339981, 0.1148313507437706, 0.014119015075266361, 0.23477430641651154, 0.6911886930465698, 0.841257631778717, 0.8693640232086182, 1.002920389175415]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Monter", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 64998], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 47.4112663269043, 60.84828186035156, 99.16316223144531, 218.51290893554688, 91.53726959228516, 119.7630615234375, 16.603445053100586, 229.5046844482422, 786.9169921875, 737.2642822265625, 953.1492919921875, 1015.8197021484375]], "actual_norm": [[398.5599365234375, 47.43720626831055, 60.84493637084961, 99.12416076660156, 218.51722717285156, 91.56475830078125, 119.74382019042969, 16.661346435546875, 229.51002502441406, 786.9428100585938, 737.33935546875, 953.153564453125, 1015.9100952148438]], "relative_norm": [[0.025199946016073227, 0.06060829386115074, 0.05266714096069336, 0.0875743180513382, 0.20548823475837708, 0.08485085517168045, 0.11560573428869247, 0.016745032742619514, 0.23878903687000275, 0.6926319599151611, 0.8358803987503052, 0.8769122958183289, 1.0065933465957642]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 23279, 6633, 3639, 17692, 11192, 19551, 220, 84147, 5313], "first_top_logits": [29.625, 22.625, 21.25, 20.5, 20.25, 19.75, 19.75, 19.375, 19.0, 19.0], "source_correct": false, "target_correct": false, "seconds": 0.4328281250782311, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.45212769508361816, "target_logp": -38.54764175415039, "log_odds": -38.09551405906677, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-0.2911803424358368, -0.16094735264778137]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-37.886741638183594, -0.6603856086730957, -0.0005133026279509068]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67239379882812, 782.2499389648438, 739.340576171875, 965.5000610351562, 1015.1803588867188]], "actual_norm": [[398.4502868652344, 37.473392486572266, 54.670570373535156, 103.79913330078125, 212.8820343017578, 87.15972137451172, 117.54987335205078, 15.357227325439453, 224.64036560058594, 782.2875366210938, 739.226806640625, 965.4044799804688, 1015.301025390625]], "relative_norm": [[0.02519301325082779, 0.0472940169274807, 0.04719533398747444, 0.09154237806797028, 0.2005847990512848, 0.08062568306922913, 0.11330492049455643, 0.015473878011107445, 0.2335953712463379, 0.6892870664596558, 0.8386222124099731, 0.8840330839157104, 1.0082650184631348]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 782.5341186523438, 742.5804443359375, 946.0936889648438, 1015.026123046875]], "actual_norm": [[398.4502868652344, 38.8733024597168, 55.16315841674805, 102.55784606933594, 215.57412719726562, 89.64132690429688, 119.23844146728516, 14.064671516418457, 225.2116241455078, 782.5093994140625, 742.584716796875, 946.1067504882812, 1015.1204223632812]], "relative_norm": [[0.02519301325082779, 0.0489378497004509, 0.04772769287228584, 0.0903397873044014, 0.20297561585903168, 0.08291501551866531, 0.11483444273471832, 0.014178365468978882, 0.23479872941970825, 0.6911571621894836, 0.8412512540817261, 0.8693721294403076, 1.003020167350769]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Mexican peso.  What is the currency used in the country", "output_ids": [23279, 36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 47.4112663269043, 60.8482780456543, 99.16316223144531, 218.51290893554688, 91.53727722167969, 119.7630615234375, 16.603445053100586, 229.5046844482422, 786.9169921875, 737.2642822265625, 953.1492919921875, 1015.8197021484375]], "actual_norm": [[398.4502868652344, 47.42911911010742, 60.85100173950195, 99.18693542480469, 218.51634216308594, 91.50778198242188, 119.70364379882812, 16.614612579345703, 229.50341796875, 786.9784545898438, 737.2555541992188, 953.1138305664062, 1015.8878173828125]], "relative_norm": [[0.02519301325082779, 0.06059795990586281, 0.05267238989472389, 0.0876297727227211, 0.20548740029335022, 0.08479805290699005, 0.11556694656610489, 0.016698064282536507, 0.23878216743469238, 0.6926633715629578, 0.8357853889465332, 0.8768757581710815, 1.0065712928771973]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23279, 12270, 109448, 36935, 386, 393, 68831, 51565, 5271, 23273], "first_top_logits": [20.875, 19.5, 15.3125, 14.5, 14.0, 13.5, 13.375, 13.25, 12.6875, 12.625], "source_correct": true, "target_correct": false, "seconds": 0.4339809170924127, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.8844741582870483, "target_logp": -13.342791557312012, "log_odds": -11.458317399024963, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-1.6317723989486694, -0.2527017295360565]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-12.705154418945312, -0.637363076210022, -0.0002735478919930756]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.5189514160156, 37.46458435058594, 54.624298095703125, 103.82354736328125, 212.84266662597656, 87.18940734863281, 117.56121826171875, 15.341843605041504, 224.67237854003906, 782.2498779296875, 739.3405151367188, 965.5, 1015.1802978515625]], "actual_norm": [[398.0365905761719, 37.505008697509766, 54.59080123901367, 103.84636688232422, 212.8959197998047, 87.22409057617188, 117.60785675048828, 15.394700050354004, 224.7423858642578, 782.308837890625, 739.3436889648438, 965.661376953125, 1015.1160888671875]], "relative_norm": [[0.02516685612499714, 0.04733391851186752, 0.04712647199630737, 0.09158403426408768, 0.200597882270813, 0.08068522810935974, 0.11336081475019455, 0.015511635690927505, 0.23370146751403809, 0.68930584192276, 0.8387548327445984, 0.8842683434486389, 1.0080814361572266]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.5189514160156, 38.81449890136719, 55.18267059326172, 102.51004028320312, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.21763610839844, 782.5341186523438, 742.5804443359375, 946.0936279296875, 1015.0260620117188]], "actual_norm": [[398.0365905761719, 38.82816696166992, 55.17979431152344, 102.4885025024414, 215.63938903808594, 89.74369812011719, 119.29503631591797, 14.030046463012695, 225.2429656982422, 782.5962524414062, 742.5866088867188, 946.0838623046875, 1015.0187377929688]], "relative_norm": [[0.02516685612499714, 0.04888102784752846, 0.0477420873939991, 0.09027870744466782, 0.2030370682477951, 0.08300970494747162, 0.11488895118236542, 0.014143460430204868, 0.23483140766620636, 0.6912338733673096, 0.841253399848938, 0.8693510890007019, 1.0029196739196777]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " United States dollar.  What is the currency used in the", "output_ids": [3639, 4180, 17692, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.5189514160156, 47.41126251220703, 60.84827423095703, 99.16316223144531, 218.5128936767578, 91.53726959228516, 119.76305389404297, 16.603445053100586, 229.50466918945312, 786.9169311523438, 737.2642211914062, 953.1492309570312, 1015.8197021484375]], "actual_norm": [[398.0365905761719, 47.454856872558594, 60.78096389770508, 99.19583129882812, 218.5424346923828, 91.64020538330078, 119.85142517089844, 16.653148651123047, 229.55166625976562, 786.8643188476562, 737.3621826171875, 953.1647338867188, 1015.8182373046875]], "relative_norm": [[0.02516685612499714, 0.06063084676861763, 0.05261176824569702, 0.08763763308525085, 0.2055119425058365, 0.08492076396942139, 0.11570962518453598, 0.016736794263124466, 0.23883236944675446, 0.6925628781318665, 0.8359062671661377, 0.8769225478172302, 1.006502389907837]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [3639, 23279, 17672, 17692, 2274, 19461, 15154, 36935, 19551, 30], "first_top_logits": [21.125, 21.125, 20.875, 20.75, 20.25, 19.5, 19.125, 18.625, 18.375, 17.75], "source_correct": false, "target_correct": false, "seconds": 0.42087600007653236, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.4127055406570435, "target_logp": -16.189287185668945, "log_odds": -14.776581645011902, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-1.251891016960144, -0.16081449389457703]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-15.584144592285156, -0.6047379970550537, -0.00040344204171560705]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82356262207031, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.341845512390137, 224.6724090576172, 782.2499389648438, 739.3406372070312, 965.5001220703125, 1015.180419921875]], "actual_norm": [[398.3287658691406, 37.52971267700195, 54.63896942138672, 103.90719604492188, 212.8096466064453, 87.1723861694336, 117.6007080078125, 15.399110794067383, 224.6602020263672, 782.3226928710938, 739.3539428710938, 965.4622802734375, 1015.33251953125]], "relative_norm": [[0.02518532983958721, 0.04736509546637535, 0.047168053686618805, 0.09163767844438553, 0.20051659643650055, 0.08063740283250809, 0.1133539229631424, 0.01551607996225357, 0.23361600935459137, 0.6893180012702942, 0.8387664556503296, 0.884086012840271, 1.0082963705062866]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.182674407958984, 102.51004791259766, 215.6083526611328, 89.64251708984375, 119.22750854492188, 13.993583679199219, 225.2176513671875, 782.5341186523438, 742.5804443359375, 946.09375, 1015.0261840820312]], "actual_norm": [[398.3287658691406, 38.802337646484375, 55.16326141357422, 102.57279968261719, 215.58547973632812, 89.67176818847656, 119.27616882324219, 14.047698020935059, 225.2303924560547, 782.5491333007812, 742.661865234375, 946.100341796875, 1015.1815795898438]], "relative_norm": [[0.02518532983958721, 0.04884851351380348, 0.047727782279253006, 0.09035295993089676, 0.20298629999160767, 0.08294317871332169, 0.11487077921628952, 0.014161255210638046, 0.23481829464435577, 0.6911922693252563, 0.8413386344909668, 0.8693662285804749, 1.0030806064605713]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Mexican peso.  What is the currency used in the country", "output_ids": [23279, 36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 47.41127014160156, 60.84828186035156, 99.16316986083984, 218.51290893554688, 91.53728485107422, 119.7630615234375, 16.603445053100586, 229.50469970703125, 786.9170532226562, 737.2643432617188, 953.1492919921875, 1015.8197631835938]], "actual_norm": [[398.3287658691406, 47.430145263671875, 60.83376693725586, 99.1147232055664, 218.5565948486328, 91.5653076171875, 119.79732513427734, 16.583036422729492, 229.5646209716797, 786.9572143554688, 737.2686157226562, 953.1600952148438, 1015.8239135742188]], "relative_norm": [[0.02518532983958721, 0.06059927120804787, 0.05265747383236885, 0.08756597340106964, 0.2055252641439438, 0.08485136181116104, 0.11565738916397095, 0.01666632853448391, 0.2388458400964737, 0.6926446557044983, 0.8358001708984375, 0.8769183158874512, 1.0065079927444458]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23279, 393, 17692, 36935, 84147, 12270, 3639, 547, 304, 12137], "first_top_logits": [19.75, 19.625, 18.375, 17.75, 17.75, 17.5, 17.5, 17.0, 16.75, 16.75], "source_correct": true, "target_correct": false, "seconds": 0.4279779172502458, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-09/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-09", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Monterrey", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Monterrey is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.2082177996635437, "target_logp": -20.32073211669922, "log_odds": -20.112514317035675, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-0.10737104713916779, -0.10084674507379532]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-19.764814376831055, -0.555557370185852, -0.000360066449502483]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 37.46458435058594, 54.62430191040039, 103.82355499267578, 212.8426513671875, 87.18940734863281, 117.56121826171875, 15.341843605041504, 224.67237854003906, 782.2498779296875, 739.3405151367188, 965.5, 1015.1803588867188]], "actual_norm": [[398.0928039550781, 37.52130126953125, 54.57176208496094, 103.74887084960938, 212.82229614257812, 87.19469451904297, 117.5843276977539, 15.444535255432129, 224.6737060546875, 782.1159057617188, 739.34375, 965.61962890625, 1015.2559204101562]], "relative_norm": [[0.02517041005194187, 0.04735448211431503, 0.04711003601551056, 0.09149804711341858, 0.20052851736545563, 0.08065803349018097, 0.11333813518285751, 0.015561849810183048, 0.23363004624843597, 0.6891358494758606, 0.8387548923492432, 0.8842300772666931, 1.0082203149795532]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.2176513671875, 782.5340576171875, 742.5804443359375, 946.0936279296875, 1015.0260620117188]], "actual_norm": [[398.0928039550781, 38.86317443847656, 55.131107330322266, 102.53495788574219, 215.53646850585938, 89.6104965209961, 119.26773834228516, 14.071955680847168, 225.22122192382812, 782.5075073242188, 742.5943603515625, 945.9719848632812, 1015.0562744140625]], "relative_norm": [[0.02517041005194187, 0.04892510175704956, 0.04769996181130409, 0.09031962603330612, 0.20294015109539032, 0.08288650214672089, 0.11486265808343887, 0.014185708947479725, 0.23480874300003052, 0.6911554932594299, 0.84126216173172, 0.8692483305931091, 1.002956748008728]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Mexican peso.  What is the currency used in the country", "output_ids": [23279, 36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 64998, 8073, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[398.51898193359375, 47.4112663269043, 60.84827423095703, 99.16316223144531, 218.5128936767578, 91.53726959228516, 119.76305389404297, 16.603445053100586, 229.5046844482422, 786.9169921875, 737.2642822265625, 953.1492309570312, 1015.8197021484375]], "actual_norm": [[398.0928039550781, 47.42302703857422, 60.786949157714844, 99.18421936035156, 218.50152587890625, 91.54248046875, 119.7531509399414, 16.661426544189453, 229.44639587402344, 786.873779296875, 737.3789672851562, 953.1970825195312, 1015.82421875]], "relative_norm": [[0.02517041005194187, 0.06059017777442932, 0.052616946399211884, 0.08762737363576889, 0.20547346770763397, 0.08483020961284637, 0.11561474204063416, 0.01674511283636093, 0.23872284591197968, 0.692571222782135, 0.835925281047821, 0.8769523501396179, 1.0065083503723145]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23279, 36935, 3639, 547, 12270, 17692, 84147, 11888, 19551, 2274], "first_top_logits": [23.75, 20.875, 19.875, 19.375, 19.125, 18.0, 17.125, 16.375, 16.375, 16.25], "source_correct": true, "target_correct": false, "seconds": 0.42975025018677115, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/readout", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 210.0, "target_rank": 1004.5, "source_logit": 4.375, "target_logit": 2.640625, "top_ids": [15973, 11413, 38352, 3693, 45418, 8330, 56181, 31491, 6364, 14042, 7065, 11888], "top_tokens": ["currency", " currency", "uenta", " American", ".currency", " Australia", " Philippine", "uesto", " English", "razil", "ican", " Canadian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 30.0, "target_rank": 403.0, "source_logit": 7.3125, "target_logit": 3.96875, "top_ids": [11413, 15973, 96694, 3001, 3693, 7065, 53661, 28453, 4787, 15154, 18174, 21295], "top_tokens": [" currency", "currency", "onis", "ica", " American", "ican", "cla", " Currency", " North", " Spanish", "oret", "ollar"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 293.0, "target_rank": 550.5, "source_logit": 15.375, "target_logit": 13.9375, "top_ids": [11413, 35602, 90475, 45418, 7214, 28453, 34274, 77186, 11192, 24357, 32488, 43479], "top_tokens": [" currency", " currencies", "(currency", ".currency", " foreign", " Currency", " Caribbean", ".Currency", " dollars", " overseas", " euros", " franc"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 42.0, "target_rank": 153.0, "source_logit": 16.25, "target_logit": 14.25, "top_ids": [11413, 7214, 65001, 35602, 47412, 34710, 15973, 16675, 5313, 3220, 21130, 17692], "top_tokens": [" currency", " foreign", "\"\u2014", " currencies", " foreigners", "_currency", "currency", " nations", " national", " money", " parliament", " dollar"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 2448.5, "target_rank": 3114.0, "source_logit": 1.34375, "target_logit": 1.03125, "top_ids": [21251, 28453, 3776, 603, 3731, 7748, 56537, 5737, 4229, 19833, 11413, 55964], "top_tokens": ["onia", " Currency", " One", "one", " Red", " Indian", " _______,", " Group", " Ge", " Amb", " currency", "\u2011"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 206.0, "target_rank": 1069.5, "source_logit": 4.125, "target_logit": 2.109375, "top_ids": [11766, 18174, 11413, 364, 7179, 479, 2932, 467, 356, 362, 21251, 105469], "top_tokens": [" Standard", "oret", " currency", " '", " International", " G", " She", " W", " C", " A", "onia", "\u79cd\u7c7b"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 687.5, "target_rank": 5379.0, "source_logit": 14.125, "target_logit": 10.8125, "top_ids": [11413, 35602, 13867, 5313, 3146, 3283, 28453, 11192, 30176, 7576, 7214, 5837], "top_tokens": [" currency", " currencies", " county", " national", " country", " city", " Currency", " dollars", " counties", " prices", " foreign", " countries"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 40.0, "target_rank": 448.0, "source_logit": 15.625, "target_logit": 12.8125, "top_ids": [11413, 7214, 35602, 94936, 37198, 105492, 5313, 5837, 114141, 43479, 17672, 6489], "top_tokens": [" currency", " foreign", " currencies", " denomination", " Commonwealth", "\u4e2d\u534e\u4eba\u6c11", " national", " countries", "\u81ea\u7531\u8d38\u6613", " franc", " euro", " international"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 260.0, "target_rank": 7105.5, "source_logit": 4.125, "target_logit": -0.56640625, "top_ids": [3693, 4787, 279, 89478, 4229, 5434, 15503, 2240, 1083, 46427, 12022, 21251], "top_tokens": [" American", " North", " the", "\u2026\n", " Ge", "\u2026\n\n", " primarily", " None", " also", "/is", " \u2026\n\n", "onia"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 161.0, "target_rank": 2412.0, "source_logit": 4.90625, "target_logit": 1.0859375, "top_ids": [1852, 3693, 4102, 5938, 3639, 4787, 271, 4882, 479, 5055, 362, 2274], "top_tokens": [" same", " American", "\u00a0", " German", " United", " North", "\n\n", " South", " G", " National", " A", " US"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 881.0, "target_rank": 2112.5, "source_logit": 10.6875, "target_logit": 8.9375, "top_ids": [11413, 104142, 35602, 103408, 28453, 113883, 101237, 90475, 32488, 26321, 101406, 34710], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", "\u6b27\u5143", " Currency", "\u672c\u56fd", "\u7f8e\u5143", "(currency", " euros", "Currency", "\u8d27\u5e01", "_currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 528.0, "target_rank": 5470.0, "source_logit": 6.28125, "target_logit": 2.0625, "top_ids": [104142, 112397, 116866, 101237, 99599, 109442, 108594, 106333, 104773, 103408, 102120, 113883], "top_tokens": ["\u4eba\u6c11\u5e01", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u7f8e\u5143", "\u56fd\u5bb6", "\u8d22\u653f\u90e8", "\u53bf\u7ea7", "\u5916\u6c47", "\u5171\u548c\u56fd", "\u6b27\u5143", "\u8d22\u653f", "\u672c\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 13.5, "target_rank": 1498.5, "source_logit": 17.375, "target_logit": 4.1875, "top_ids": [11413, 11192, 17692, 104142, 23279, 19551, 80189, 42364, 28453, 101237, 32488, 2598], "top_tokens": [" currency", " dollars", " dollar", "\u4eba\u6c11\u5e01", " Mexican", " USD", " Dollars", " Dollar", " Currency", "\u7f8e\u5143", " euros", " called"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 9.0, "target_rank": 1012.5, "source_logit": 18.375, "target_logit": 4.3125, "top_ids": [23279, 11413, 104142, 17692, 15154, 19551, 11192, 3639, 12270, 42364, 28453, 101237], "top_tokens": [" Mexican", " currency", "\u4eba\u6c11\u5e01", " dollar", " Spanish", " USD", " dollars", " United", " Mexico", " Dollar", " Currency", "\u7f8e\u5143"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 30.0, "target_rank": 1253.0, "source_logit": 28.75, "target_logit": 11.4375, "top_ids": [11192, 11413, 104142, 17692, 101237, 80189, 35602, 28453, 42364, 103408, 19551, 101406], "top_tokens": [" dollars", " currency", "\u4eba\u6c11\u5e01", " dollar", "\u7f8e\u5143", " Dollars", " currencies", " Currency", " Dollar", "\u6b27\u5143", " USD", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 18.0, "target_rank": 1735.5, "source_logit": 23.125, "target_logit": 5.90625, "top_ids": [104142, 101237, 23279, 17692, 11413, 84147, 101406, 109448, 11192, 35602, 28453, 19551], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", " Mexican", " dollar", " currency", " pesos", "\u8d27\u5e01", "\u58a8\u897f\u54e5", " dollars", " currencies", " Currency", " USD"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 13.5, "target_rank": 18354.0, "source_logit": 20.25, "target_logit": 3.25, "top_ids": [279, 36935, 3881, 2598, 3639, 23279, 56856, 393, 2274, 17692, 84147, 11192], "top_tokens": [" the", " peso", " known", " called", " United", " Mexican", " Pes", " P", " US", " dollar", " pesos", " dollars"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 10.0, "target_rank": 54069.5, "source_logit": 20.25, "target_logit": -2.078125, "top_ids": [36935, 393, 23279, 3639, 2274, 17692, 84147, 56856, 281, 12270, 109448, 547], "top_tokens": [" peso", " P", " Mexican", " United", " US", " dollar", " pesos", " Pes", " p", " Mexico", "\u58a8\u897f\u54e5", " U"]}], "source_token": 12270, "target_token": 8359, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Gu", "adal", "aj", "ara", " is", " the"], "seconds": 1.1651929579675198}
{"run_id": "eval-10/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.1670379638671875, "target_logp": -23.46457290649414, "log_odds": -20.297534942626953, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-3.1115493774414062, -0.05548865348100662]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-23.142799377441406, -0.3215843141078949, -0.00018857131362892687]}}, "score_trace": {"source": [], "target": []}, "output": " peso.  What is the currency of the country containing Gu", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4570], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [], "first_top_ids": [36935, 393, 23279, 3639, 2274, 17692, 84147, 56856, 281, 12270], "first_top_logits": [29.625, 26.875, 26.75, 23.625, 22.125, 21.75, 21.0, 20.5, 20.375, 20.25], "source_correct": true, "target_correct": false, "seconds": 0.4910875828936696, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.1670379638671875, "target_logp": -23.46457290649414, "log_odds": -20.297534942626953, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-3.1115493774414062, -0.05548865348100662]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-23.142799377441406, -0.3215843141078949, -0.00018857131362892687]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency of the country containing Gu", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4570], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 393, 23279, 3639, 2274, 17692, 84147, 56856, 281, 12270], "first_top_logits": [29.625, 26.875, 26.75, 23.625, 22.125, 21.75, 21.0, 20.5, 20.375, 20.25], "source_correct": true, "target_correct": false, "seconds": 0.4613617919385433, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -37.836910247802734, "target_logp": -1.9434552192687988, "log_odds": 35.893455028533936, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-37.655975341796875, -0.18093521893024445]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-1.8122267723083496, -0.13095587491989136, -0.00027247529942542315]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.633056640625]], "actual_norm": [[398.5150451660156, 38.854148864746094, 55.21453094482422, 102.5206527709961, 215.6384735107422, 89.64886474609375, 119.22421264648438, 14.077310562133789, 225.23284912109375, 570.3524780273438, 776.9832763671875, 784.461181640625, 774.6055297851562, 818.9052734375, 860.6495971679688]], "relative_norm": [[0.0251971073448658, 0.04891373962163925, 0.04777214303612709, 0.09030702710151672, 0.20303620398044586, 0.08292198926210403, 0.11482074111700058, 0.014191106893122196, 0.23482085764408112, 0.5248518586158752, 0.6486905217170715, 0.7124147415161133, 0.8699067234992981, 0.7681137323379517, 0.8719332814216614]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.633056640625]], "actual_norm": [[398.5150451660156, 38.854148864746094, 55.21453094482422, 102.5206527709961, 215.6384735107422, 89.64886474609375, 119.22421264648438, 14.077310562133789, 225.23284912109375, 570.3524780273438, 776.9832763671875, 784.461181640625, 774.6055297851562, 818.9052734375, 860.6495971679688]], "relative_norm": [[0.0251971073448658, 0.04891373962163925, 0.04777214303612709, 0.09030702710151672, 0.20303620398044586, 0.08292198926210403, 0.11482074111700058, 0.014191106893122196, 0.23482085764408112, 0.5248518586158752, 0.6486905217170715, 0.7124147415161133, 0.8699067234992981, 0.7681137323379517, 0.8719332814216614]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russia ruble.  What is the currency used in the", "output_ids": [8359, 10273, 273, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67239379882812, 564.612060546875, 770.278564453125, 778.4844970703125, 769.45556640625, 807.7573852539062, 855.3131713867188]], "actual_norm": [[398.5150451660156, 37.49421691894531, 54.68024444580078, 103.8343276977539, 212.8571319580078, 87.22295379638672, 117.56754302978516, 15.33811092376709, 224.7115478515625, 564.6292724609375, 770.346435546875, 778.468994140625, 769.4786376953125, 807.8489379882812, 855.3917846679688]], "relative_norm": [[0.0251971073448658, 0.047320298850536346, 0.04720368608832359, 0.09157341718673706, 0.20056134462356567, 0.08068417757749557, 0.113321952521801, 0.015454616397619247, 0.23366940021514893, 0.5203557014465332, 0.6440097093582153, 0.7090330719947815, 0.8645191788673401, 0.7575165033340454, 0.8660651445388794]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8359, 8522, 435, 431, 3146, 10273, 3639, 4741, 104084, 11413], "first_top_logits": [29.75, 28.25, 24.625, 23.75, 20.625, 20.625, 19.75, 19.75, 19.375, 19.0], "source_correct": false, "target_correct": true, "seconds": 0.4846484581939876, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -8.342764854431152, "target_logp": -16.935331344604492, "log_odds": -8.59256649017334, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-8.253131866455078, -0.08963264524936676]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-16.753131866455078, -0.18155676126480103, -0.0006423317245207727]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993583679199219, 225.21766662597656, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.6331176757812]], "actual_norm": [[398.5599365234375, 38.83179473876953, 55.18263626098633, 102.50543212890625, 215.60459899902344, 89.62640380859375, 119.2352294921875, 14.005796432495117, 225.18820190429688, 570.388671875, 777.06787109375, 784.385009765625, 774.4888916015625, 818.8279418945312, 860.4751586914062]], "relative_norm": [[0.025199946016073227, 0.04888559505343437, 0.047744546085596085, 0.09029361605644226, 0.20300430059432983, 0.08290121704339981, 0.1148313507437706, 0.014119015075266361, 0.23477430641651154, 0.5248851776123047, 0.6487611532211304, 0.7123456001281738, 0.8697757124900818, 0.76804119348526, 0.8717565536499023]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993583679199219, 225.21766662597656, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.6331176757812]], "actual_norm": [[398.5599365234375, 38.83179473876953, 55.18263626098633, 102.50543212890625, 215.60459899902344, 89.62640380859375, 119.2352294921875, 14.005796432495117, 225.18820190429688, 570.388671875, 777.06787109375, 784.385009765625, 774.4888916015625, 818.8279418945312, 860.4751586914062]], "relative_norm": [[0.025199946016073227, 0.04888559505343437, 0.047744546085596085, 0.09029361605644226, 0.20300430059432983, 0.08290121704339981, 0.1148313507437706, 0.014119015075266361, 0.23477430641651154, 0.5248851776123047, 0.6487611532211304, 0.7123456001281738, 0.8697757124900818, 0.76804119348526, 0.8717565536499023]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Gu", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4570], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67237854003906, 564.612060546875, 770.2786254882812, 778.4845581054688, 769.4556274414062, 807.7574462890625, 855.313232421875]], "actual_norm": [[398.5599365234375, 37.49838638305664, 54.59830093383789, 103.8289794921875, 212.8018035888672, 87.17161560058594, 117.55985260009766, 15.413719177246094, 224.66209411621094, 564.6492309570312, 770.3336791992188, 778.5679321289062, 769.405029296875, 807.7373657226562, 855.273193359375]], "relative_norm": [[0.025199946016073227, 0.047325558960437775, 0.0471329465508461, 0.09156870096921921, 0.2005092054605484, 0.08063668757677078, 0.11331453919410706, 0.015530799515545368, 0.23361797630786896, 0.5203741192817688, 0.6439990401268005, 0.7091231942176819, 0.8644365072250366, 0.7574118971824646, 0.8659451007843018]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 17692, 6633, 5313, 23279, 36935, 220, 19551, 3146, 3639], "first_top_logits": [28.75, 21.5, 21.0, 21.0, 20.625, 20.25, 20.0, 19.625, 19.5, 19.5], "source_correct": false, "target_correct": false, "seconds": 0.44914920907467604, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.24719014763832092, "target_logp": -41.623905181884766, "log_odds": -41.376715034246445, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-0.16751137375831604, -0.07967877388000488]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-41.292510986328125, -0.331005334854126, -0.00038985759601928294]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.633056640625]], "actual_norm": [[398.4502868652344, 38.8733024597168, 55.16315841674805, 102.55784606933594, 215.57412719726562, 89.64132690429688, 119.23844146728516, 14.064671516418457, 225.2116241455078, 570.401611328125, 777.0169067382812, 784.4097900390625, 774.48046875, 818.9852905273438, 860.6090698242188]], "relative_norm": [[0.02519301325082779, 0.0489378497004509, 0.04772769287228584, 0.0903397873044014, 0.20297561585903168, 0.08291501551866531, 0.11483444273471832, 0.014178365468978882, 0.23479872941970825, 0.5248970985412598, 0.6487185955047607, 0.7123680710792542, 0.8697662949562073, 0.7681888341903687, 0.8718922138214111]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22750091552734, 13.993582725524902, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4740600585938, 818.9033813476562, 860.633056640625]], "actual_norm": [[398.4502868652344, 38.8733024597168, 55.16315841674805, 102.55784606933594, 215.57412719726562, 89.64132690429688, 119.23844146728516, 14.064671516418457, 225.2116241455078, 570.401611328125, 777.0169067382812, 784.4097900390625, 774.48046875, 818.9852905273438, 860.6090698242188]], "relative_norm": [[0.02519301325082779, 0.0489378497004509, 0.04772769287228584, 0.0903397873044014, 0.20297561585903168, 0.08291501551866531, 0.11483444273471832, 0.014178365468978882, 0.23479872941970825, 0.5248970985412598, 0.6487185955047607, 0.7123680710792542, 0.8697662949562073, 0.7681888341903687, 0.8718922138214111]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Mexican peso.  What is the currency used in the country", "output_ids": [23279, 36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82355499267578, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.34184455871582, 224.67239379882812, 564.612060546875, 770.278564453125, 778.4844970703125, 769.45556640625, 807.7573852539062, 855.3131713867188]], "actual_norm": [[398.4502868652344, 37.473392486572266, 54.670570373535156, 103.79913330078125, 212.8820343017578, 87.15972137451172, 117.54987335205078, 15.357227325439453, 224.64036560058594, 564.5801391601562, 770.1680908203125, 778.5326538085938, 769.4318237304688, 807.7093505859375, 855.30419921875]], "relative_norm": [[0.02519301325082779, 0.0472940169274807, 0.04719533398747444, 0.09154237806797028, 0.2005847990512848, 0.08062568306922913, 0.11330492049455643, 0.015473878011107445, 0.2335953712463379, 0.5203104615211487, 0.6438605785369873, 0.709091067314148, 0.8644666075706482, 0.7573856711387634, 0.8659764528274536]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23279, 12270, 109448, 36935, 393, 1852, 386, 23273, 5271, 9369], "first_top_logits": [22.875, 21.125, 17.375, 16.5, 15.8125, 14.5625, 14.4375, 14.25, 13.875, 13.8125], "source_correct": true, "target_correct": false, "seconds": 0.44840412493795156, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.129732608795166, "target_logp": -13.644609451293945, "log_odds": -9.51487684249878, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-4.016136169433594, -0.11359664797782898]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-13.328636169433594, -0.3158301115036011, -0.00014351768186315894]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.5189514160156, 38.81449890136719, 55.18267059326172, 102.51004028320312, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.21763610839844, 570.3903198242188, 777.0353393554688, 784.4168090820312, 774.4739990234375, 818.9033203125, 860.6329956054688]], "actual_norm": [[398.0365905761719, 38.82816696166992, 55.17979431152344, 102.4885025024414, 215.63938903808594, 89.74369812011719, 119.29503631591797, 14.030046463012695, 225.2429656982422, 570.3848266601562, 777.163330078125, 784.3932495117188, 774.489501953125, 818.8577880859375, 860.5504760742188]], "relative_norm": [[0.02516685612499714, 0.04888102784752846, 0.0477420873939991, 0.09027870744466782, 0.2030370682477951, 0.08300970494747162, 0.11488895118236542, 0.014143460430204868, 0.23483140766620636, 0.5248816609382629, 0.6488408446311951, 0.7123530507087708, 0.8697764277458191, 0.7680692076683044, 0.8718328475952148]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.5189514160156, 38.81449890136719, 55.18267059326172, 102.51004028320312, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.21763610839844, 570.3903198242188, 777.0353393554688, 784.4168090820312, 774.4739990234375, 818.9033203125, 860.6329956054688]], "actual_norm": [[398.0365905761719, 38.82816696166992, 55.17979431152344, 102.4885025024414, 215.63938903808594, 89.74369812011719, 119.29503631591797, 14.030046463012695, 225.2429656982422, 570.3848266601562, 777.163330078125, 784.3932495117188, 774.489501953125, 818.8577880859375, 860.5504760742188]], "relative_norm": [[0.02516685612499714, 0.04888102784752846, 0.0477420873939991, 0.09027870744466782, 0.2030370682477951, 0.08300970494747162, 0.11488895118236542, 0.014143460430204868, 0.23483140766620636, 0.5248816609382629, 0.6488408446311951, 0.7123530507087708, 0.8697764277458191, 0.7680692076683044, 0.8718328475952148]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  What is the currency used in the country containing", "output_ids": [17672, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.5189514160156, 37.46458435058594, 54.624298095703125, 103.82354736328125, 212.84266662597656, 87.18940734863281, 117.56121826171875, 15.341843605041504, 224.67237854003906, 564.612060546875, 770.2785034179688, 778.4844360351562, 769.4555053710938, 807.75732421875, 855.3131103515625]], "actual_norm": [[398.0365905761719, 37.505008697509766, 54.59080123901367, 103.84636688232422, 212.8959197998047, 87.22409057617188, 117.60785675048828, 15.394700050354004, 224.7423858642578, 564.573974609375, 770.2711791992188, 778.5759887695312, 769.52880859375, 807.78662109375, 855.302001953125]], "relative_norm": [[0.02516685612499714, 0.04733391851186752, 0.04712647199630737, 0.09158403426408768, 0.200597882270813, 0.08068522810935974, 0.11336081475019455, 0.015511635690927505, 0.23370146751403809, 0.5203047394752502, 0.6439467668533325, 0.7091305255889893, 0.8645755648612976, 0.7574580907821655, 0.8659742474555969]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 17692, 3639, 19461, 2274, 36935, 15154, 23279, 42364, 19551], "first_top_logits": [22.875, 22.625, 22.125, 21.875, 21.125, 21.0, 20.0, 20.0, 19.0, 18.875], "source_correct": false, "target_correct": false, "seconds": 0.45017958292737603, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.8324780464172363, "target_logp": -16.312238693237305, "log_odds": -13.479760646820068, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-2.7699453830718994, -0.06253272294998169]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-16.01994514465332, -0.2920253276824951, -0.0002696150622796267]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.182674407958984, 102.51004791259766, 215.6083526611328, 89.64251708984375, 119.22750854492188, 13.993583679199219, 225.2176513671875, 570.390380859375, 777.035400390625, 784.4168090820312, 774.47412109375, 818.9034423828125, 860.6331176757812]], "actual_norm": [[398.3287658691406, 38.802337646484375, 55.16326141357422, 102.57279968261719, 215.58547973632812, 89.67176818847656, 119.27616882324219, 14.047698020935059, 225.2303924560547, 570.365966796875, 777.0630493164062, 784.3700561523438, 774.5149536132812, 818.9197387695312, 860.5838012695312]], "relative_norm": [[0.02518532983958721, 0.04884851351380348, 0.047727782279253006, 0.09035295993089676, 0.20298629999160767, 0.08294317871332169, 0.11487077921628952, 0.014161255210638046, 0.23481829464435577, 0.5248643159866333, 0.6487571597099304, 0.712332010269165, 0.8698049783706665, 0.7681273221969604, 0.8718665838241577]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.182674407958984, 102.51004791259766, 215.6083526611328, 89.64251708984375, 119.22750854492188, 13.993583679199219, 225.2176513671875, 570.390380859375, 777.035400390625, 784.4168090820312, 774.47412109375, 818.9034423828125, 860.6331176757812]], "actual_norm": [[398.3287658691406, 38.802337646484375, 55.16326141357422, 102.57279968261719, 215.58547973632812, 89.67176818847656, 119.27616882324219, 14.047698020935059, 225.2303924560547, 570.365966796875, 777.0630493164062, 784.3700561523438, 774.5149536132812, 818.9197387695312, 860.5838012695312]], "relative_norm": [[0.02518532983958721, 0.04884851351380348, 0.047727782279253006, 0.09035295993089676, 0.20298629999160767, 0.08294317871332169, 0.11487077921628952, 0.014161255210638046, 0.23481829464435577, 0.5248643159866333, 0.6487571597099304, 0.712332010269165, 0.8698049783706665, 0.7681273221969604, 0.8718665838241577]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency used in the country containing", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 37.4645881652832, 54.62430191040039, 103.82356262207031, 212.84266662597656, 87.18940734863281, 117.56122589111328, 15.341845512390137, 224.6724090576172, 564.612060546875, 770.2786254882812, 778.4844970703125, 769.45556640625, 807.7574462890625, 855.313232421875]], "actual_norm": [[398.3287658691406, 37.52971267700195, 54.63896942138672, 103.90719604492188, 212.8096466064453, 87.1723861694336, 117.6007080078125, 15.399110794067383, 224.6602020263672, 564.619384765625, 770.240478515625, 778.409912109375, 769.4493408203125, 807.725341796875, 855.27490234375]], "relative_norm": [[0.02518532983958721, 0.04736509546637535, 0.047168053686618805, 0.09163767844438553, 0.20051659643650055, 0.08063740283250809, 0.1133539229631424, 0.01551607996225357, 0.23361600935459137, 0.5203465819358826, 0.6439211368560791, 0.7089792490005493, 0.8644862771034241, 0.757400631904602, 0.8659468293190002]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 393, 56856, 17692, 23279, 18050, 84147, 547, 3639, 281], "first_top_logits": [21.625, 21.25, 20.125, 19.625, 19.625, 19.0, 18.25, 17.625, 17.625, 17.5], "source_correct": true, "target_correct": false, "seconds": 0.4580713748000562, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-10/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-10", "split": "eval", "country": "Mexico", "currency": "Mexican peso", "aliases": ["peso", "MXN"], "city": "Guadalajara", "target_country": "Russia", "target_currency": "Russian ruble", "target_aliases": ["ruble", "RUB", "rouble"], "prompt": "Fact: The currency used in the country containing Guadalajara is the", "source_continuation": " Mexican peso", "target_continuation": " Russian ruble"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.123511552810669, "target_logp": -20.038270950317383, "log_odds": -18.914759397506714, "token_scores": {"source": {"ids": [23279, 36935], "log_probs": [-1.084901213645935, -0.038610342890024185]}, "target": {"ids": [8522, 10273, 273], "log_probs": [-19.772401809692383, -0.26562705636024475, -0.0002426806604489684]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4739990234375, 818.9033203125, 860.6329956054688]], "actual_norm": [[398.0928039550781, 38.86317443847656, 55.131107330322266, 102.53495788574219, 215.53646850585938, 89.6104965209961, 119.26773834228516, 14.071955680847168, 225.22122192382812, 570.4097290039062, 777.1416625976562, 784.4204711914062, 774.5115356445312, 818.89111328125, 860.6241455078125]], "relative_norm": [[0.02517041005194187, 0.04892510175704956, 0.04769996181130409, 0.09031962603330612, 0.20294015109539032, 0.08288650214672089, 0.11486265808343887, 0.014185708947479725, 0.23480874300003052, 0.5249045491218567, 0.6488227844238281, 0.7123777866363525, 0.8698011636734009, 0.7681004405021667, 0.8719074726104736]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 38.81450271606445, 55.18267059326172, 102.51004791259766, 215.60833740234375, 89.64251708984375, 119.22749328613281, 13.993581771850586, 225.2176513671875, 570.390380859375, 777.0353393554688, 784.4168090820312, 774.4739990234375, 818.9033203125, 860.6329956054688]], "actual_norm": [[398.0928039550781, 38.86317443847656, 55.131107330322266, 102.53495788574219, 215.53646850585938, 89.6104965209961, 119.26773834228516, 14.071955680847168, 225.22122192382812, 570.4097290039062, 777.1416625976562, 784.4204711914062, 774.5115356445312, 818.89111328125, 860.6241455078125]], "relative_norm": [[0.02517041005194187, 0.04892510175704956, 0.04769996181130409, 0.09031962603330612, 0.20294015109539032, 0.08288650214672089, 0.11486265808343887, 0.014185708947479725, 0.23480874300003052, 0.5249045491218567, 0.6488227844238281, 0.7123777866363525, 0.8698011636734009, 0.7681004405021667, 0.8719074726104736]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " peso.  What is the currency of the country containing Gu", "output_ids": [36935, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4570], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4570, 50488, 1630, 5059, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[398.51898193359375, 37.46458435058594, 54.62430191040039, 103.82355499267578, 212.8426513671875, 87.18940734863281, 117.56121826171875, 15.341843605041504, 224.67237854003906, 564.612060546875, 770.2785034179688, 778.4844360351562, 769.45556640625, 807.75732421875, 855.3131103515625]], "actual_norm": [[398.0928039550781, 37.52130126953125, 54.57176208496094, 103.74887084960938, 212.82229614257812, 87.19469451904297, 117.5843276977539, 15.444535255432129, 224.6737060546875, 564.5713500976562, 770.4002685546875, 778.591064453125, 769.4910278320312, 807.8024291992188, 855.31298828125]], "relative_norm": [[0.02517041005194187, 0.04735448211431503, 0.04711003601551056, 0.09149804711341858, 0.20052851736545563, 0.08065803349018097, 0.11333813518285751, 0.015561849810183048, 0.23363004624843597, 0.5203023552894592, 0.6440547108650208, 0.7091442942619324, 0.8645331263542175, 0.7574729323387146, 0.8659853935241699]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [36935, 23279, 3639, 547, 17692, 84147, 393, 12270, 2274, 4882], "first_top_logits": [24.625, 24.0, 20.25, 19.75, 19.625, 19.5, 19.0, 18.625, 17.75, 17.5], "source_correct": true, "target_correct": false, "seconds": 0.44275491684675217, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/readout", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 519.5, "target_rank": 564.0, "source_logit": 3.703125, "target_logit": 3.625, "top_ids": [11413, 45418, 15973, 8330, 38352, 7513, 7065, 4505, 3693, 31491, 19312, 47238], "top_tokens": [" currency", ".currency", "currency", " Australia", "uenta", " European", "ican", " Europe", " American", "uesto", "urname", " Regulation"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 222.5, "target_rank": 2179.0, "source_logit": 4.625, "target_logit": 1.8515625, "top_ids": [11413, 15973, 96694, 3001, 28453, 7065, 5088, 3693, 53661, 18174, 45418, 28130], "top_tokens": [" currency", "currency", "onis", "ica", " Currency", "ican", "urrency", " American", "cla", "oret", ".currency", "ija"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 120.5, "target_rank": 372.5, "source_logit": 17.5, "target_logit": 15.1875, "top_ids": [11413, 35602, 90475, 7513, 32488, 24357, 22956, 36445, 7214, 6489, 85466, 8330], "top_tokens": [" currency", " currencies", "(currency", " European", " euros", " overseas", "\".\n", " internationally", " foreign", " international", " Euros", " Australia"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 102.0, "target_rank": 1041.5, "source_logit": 15.0, "target_logit": 11.375, "top_ids": [11413, 65001, 7214, 47412, 35602, 21130, 37198, 13224, 7855, 74508, 78253, 34710], "top_tokens": [" currency", "\"\u2014", " foreign", " foreigners", " currencies", " parliament", " Commonwealth", " Britain", " British", "\u00bb.", " treasury", "_currency"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 3072.5, "target_rank": 4703.5, "source_logit": 1.21875, "target_logit": 0.6328125, "top_ids": [21251, 603, 55964, 3776, 12751, 3731, 7748, 29921, 67476, 28453, 5737, 56537], "top_tokens": ["onia", "one", "\u2011", " One", " Roman", " Red", " Indian", "\u2010", "'),('", " Currency", " Group", " _______,"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 828.0, "target_rank": 2577.0, "source_logit": 2.828125, "target_logit": 1.2109375, "top_ids": [18174, 479, 21251, 2932, 7179, 364, 11766, 11413, 362, 4102, 467, 64], "top_tokens": ["oret", " G", "onia", " She", " International", " '", " Standard", " currency", " A", "\u00a0", " W", "a"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 2346.0, "target_rank": 1000.5, "source_logit": 12.0, "target_logit": 13.25, "top_ids": [11413, 35602, 13867, 5313, 3146, 3283, 6290, 28453, 30176, 7576, 29970, 7214], "top_tokens": [" currency", " currencies", " county", " national", " country", " city", " town", " Currency", " counties", " prices", " tourist", " foreign"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 225.0, "target_rank": 1016.5, "source_logit": 13.75, "target_logit": 11.75, "top_ids": [11413, 37198, 94936, 35602, 7855, 7214, 5313, 105492, 6364, 6489, 86049, 16847], "top_tokens": [" currency", " Commonwealth", " denomination", " currencies", " British", " foreign", " national", "\u4e2d\u534e\u4eba\u6c11", " English", " international", " monarchy", " province"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 8275.0, "target_rank": 10286.0, "source_logit": -0.447265625, "target_logit": -0.83984375, "top_ids": [3693, 4787, 21251, 5434, 111012, 89478, 15503, 279, 4102, 2240, 4229, 271], "top_tokens": [" American", " North", "onia", "\u2026\n\n", "\u4e16\u754c\u4e0a\u6700", "\u2026\n", " primarily", " the", "\u00a0", " None", " Ge", "\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 2386.5, "target_rank": 4513.5, "source_logit": 1.2421875, "target_logit": 0.263671875, "top_ids": [1852, 4102, 4787, 479, 5938, 3693, 4882, 18174, 3639, 271, 362, 5055], "top_tokens": [" same", "\u00a0", " North", " G", " German", " American", " South", "oret", " United", "\n\n", " A", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1781.0, "target_rank": 5063.5, "source_logit": 6.78125, "target_logit": 4.71875, "top_ids": [103408, 104142, 11413, 35602, 113883, 28453, 32488, 26321, 90475, 113596, 106333, 85466], "top_tokens": ["\u6b27\u5143", "\u4eba\u6c11\u5e01", " currency", " currencies", "\u672c\u56fd", " Currency", " euros", "Currency", "(currency", "\u56fd\u7c4d", "\u5916\u6c47", " Euros"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 4360.0, "target_rank": 18242.0, "source_logit": 2.15625, "target_logit": -0.59375, "top_ids": [116866, 104142, 112397, 109442, 103408, 101237, 106333, 101406, 99599, 102120, 110065, 113883], "top_tokens": ["\u76f4\u8f96\u5e02", "\u4eba\u6c11\u5e01", "\u56fd\u5bb6\u6807\u51c6", "\u8d22\u653f\u90e8", "\u6b27\u5143", "\u7f8e\u5143", "\u5916\u6c47", "\u8d27\u5e01", "\u56fd\u5bb6", "\u8d22\u653f", "\u6c47\u7387", "\u672c\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 51.0, "target_rank": 2384.0, "source_logit": 13.0625, "target_logit": 3.546875, "top_ids": [11413, 11192, 17692, 104142, 28453, 2598, 19551, 8522, 32488, 35602, 80189, 112455], "top_tokens": [" currency", " dollars", " dollar", "\u4eba\u6c11\u5e01", " Currency", " called", " USD", " Russian", " euros", " currencies", " Dollars", "\u82f1\u9551"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 8.0, "target_rank": 1256.0, "source_logit": 18.0, "target_logit": 4.21875, "top_ids": [8522, 11413, 104084, 19390, 104142, 17692, 47707, 8359, 28453, 19551, 22415, 11192], "top_tokens": [" Russian", " currency", "\u4fc4\u7f57\u65af", " Soviet", "\u4eba\u6c11\u5e01", " dollar", "Russian", " Russia", " Currency", " USD", " Moscow", " dollars"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 58.5, "target_rank": 1544.0, "source_logit": 20.0, "target_logit": 8.75, "top_ids": [11413, 104142, 11192, 35602, 28453, 17692, 103408, 112455, 101406, 80189, 32488, 77186], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " currencies", " Currency", " dollar", "\u6b27\u5143", "\u82f1\u9551", "\u8d27\u5e01", " Dollars", " euros", ".Currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 16.5, "target_rank": 1474.0, "source_logit": 19.25, "target_logit": 4.875, "top_ids": [104142, 8522, 11413, 104084, 35602, 28453, 47707, 45418, 101406, 77186, 17692, 101237], "top_tokens": ["\u4eba\u6c11\u5e01", " Russian", " currency", "\u4fc4\u7f57\u65af", " currencies", " Currency", "Russian", ".currency", "\u8d27\u5e01", ".Currency", " dollar", "\u7f8e\u5143"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 59.5, "target_rank": 7917.5, "source_logit": 16.75, "target_logit": 5.53125, "top_ids": [279, 8522, 2598, 3881, 10273, 7855, 34974, 220, 2176, 5644, 305, 2987], "top_tokens": [" the", " Russian", " called", " known", " rub", " British", " Kaz", " ", " both", " held", " h", " either"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 7.0, "target_rank": 5138.5, "source_logit": 19.375, "target_logit": 4.71875, "top_ids": [8522, 10273, 34974, 431, 12839, 435, 8359, 94705, 104084, 11413, 220, 19390], "top_tokens": [" Russian", " rub", " Kaz", " R", " Rub", " r", " Russia", " Cyril", "\u4fc4\u7f57\u65af", " currency", " ", " Soviet"]}], "source_token": 8359, "target_token": 17009, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Kaz", "an", " is", " the"], "seconds": 1.1548546659760177}
{"run_id": "eval-11/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.06664131581783295, "target_logp": -18.271677017211914, "log_odds": -18.20503570139408, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.0011798333143815398, -0.0654035434126854, -5.793403761344962e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-18.18868064880371, -0.07958916574716568, -0.0034058440942317247]}}, "score_trace": {"source": [], "target": []}, "output": " Russian ruble.  Inference: The country is Russia", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 3146, 374, 8359], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [], "first_top_ids": [8522, 10273, 34974, 12839, 431, 435, 8359, 94705, 104084, 11413], "first_top_logits": [30.125, 22.75, 21.125, 21.0, 21.0, 20.375, 19.375, 18.875, 18.625, 18.375], "source_correct": true, "target_correct": false, "seconds": 0.4940744158811867, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.06664131581783295, "target_logp": -18.271677017211914, "log_odds": -18.20503570139408, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.0011798333143815398, -0.0654035434126854, -5.793403761344962e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-18.18868064880371, -0.07958916574716568, -0.0034058440942317247]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The country is Russia", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 3146, 374, 8359], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 10273, 34974, 12839, 431, 435, 8359, 94705, 104084, 11413], "first_top_logits": [30.125, 22.75, 21.125, 21.0, 21.0, 20.375, 19.375, 18.875, 18.625, 18.375], "source_correct": true, "target_correct": false, "seconds": 0.44912516698241234, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -37.864444732666016, "target_logp": -3.768751621246338, "log_odds": 34.09569311141968, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-37.71397018432617, -0.15039454400539398, -8.165503095369786e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-3.6514713764190674, -0.11326832324266434, -0.004011916927993298]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.9727783203125, 83.60452270507812, 25.083982467651367, 205.33782958984375, 195.2326202392578, 248.97811889648438, 145.00331115722656, 94.5683364868164, 94.02424621582031, 84.24742889404297, 549.380126953125, 511.99554443359375, 752.5233154296875]], "relative_norm": [[0.004930023569613695, 0.1052502691745758, 0.021702902391552925, 0.1808752417564392, 0.183822900056839, 0.23029585182666779, 0.1396477073431015, 0.0953327938914299, 0.09802679717540741, 0.06637939065694809, 0.5991724133491516, 0.4633793532848358, 0.7276913523674011]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.9727783203125, 83.60452270507812, 25.083982467651367, 205.33782958984375, 195.2326202392578, 248.97811889648438, 145.00331115722656, 94.5683364868164, 94.02424621582031, 84.24742889404297, 549.380126953125, 511.99554443359375, 752.5233154296875]], "relative_norm": [[0.004930023569613695, 0.1052502691745758, 0.021702902391552925, 0.1808752417564392, 0.183822900056839, 0.23029585182666779, 0.1396477073431015, 0.0953327938914299, 0.09802679717540741, 0.06637939065694809, 0.5991724133491516, 0.4633793532848358, 0.7276913523674011]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkeylite.  What is the currency used in the country", "output_ids": [17009, 68078, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.6080551147461, 23.151418685913086, 204.158203125, 193.7395782470703, 247.857666015625, 142.53195190429688, 101.13684844970703, 101.15641784667969, 80.84549713134766, 533.1124267578125, 500.5370178222656, 718.2415161132812]], "actual_norm": [[77.9727783203125, 92.63127136230469, 23.218753814697266, 204.1670379638672, 193.71022033691406, 247.91635131835938, 142.5072479248047, 101.14851379394531, 101.17664337158203, 80.87863159179688, 533.1513671875, 500.5521545410156, 718.3020629882812]], "relative_norm": [[0.004930023569613695, 0.11835063248872757, 0.020098064094781876, 0.18037770688533783, 0.18216033279895782, 0.22973810136318207, 0.13758251070976257, 0.10165656358003616, 0.10526718199253082, 0.06346845626831055, 0.5796384215354919, 0.45532333850860596, 0.6978868842124939]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17009, 23734, 350, 44305, 356, 8705, 109236, 69010, 72719, 20545], "first_top_logits": [29.125, 25.75, 19.375, 18.75, 18.375, 18.25, 18.0, 17.875, 17.25, 16.875], "source_correct": false, "target_correct": false, "seconds": 0.45080074993893504, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.16025079786777496, "target_logp": -15.770702362060547, "log_odds": -15.610451564192772, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.11465112864971161, -0.04543266445398331, -0.00016699827392585576]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-15.67715072631836, -0.0894109308719635, -0.004140377044677734]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.595947265625, 94.04041290283203, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.87776947021484, 83.6090087890625, 25.066511154174805, 205.3695068359375, 195.1577911376953, 248.9435272216797, 145.02371215820312, 94.62918853759766, 94.04428100585938, 84.30207061767578, 549.3426513671875, 511.9051208496094, 752.4816284179688]], "relative_norm": [[0.004924016073346138, 0.10525591671466827, 0.021687785163521767, 0.18090315163135529, 0.18375244736671448, 0.2302638441324234, 0.139667347073555, 0.09539414197206497, 0.09804768115282059, 0.06642244011163712, 0.5991315841674805, 0.4632975161075592, 0.727651059627533]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.595947265625, 94.04041290283203, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.87776947021484, 83.6090087890625, 25.066511154174805, 205.3695068359375, 195.1577911376953, 248.9435272216797, 145.02371215820312, 94.62918853759766, 94.04428100585938, 84.30207061767578, 549.3426513671875, 511.9051208496094, 752.4816284179688]], "relative_norm": [[0.004924016073346138, 0.10525591671466827, 0.021687785163521767, 0.18090315163135529, 0.18375244736671448, 0.2302638441324234, 0.139667347073555, 0.09539414197206497, 0.09804768115282059, 0.06642244011163712, 0.5991315841674805, 0.4632975161075592, 0.727651059627533]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The country is Russia", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 3146, 374, 8359], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.6080551147461, 23.151418685913086, 204.158203125, 193.7395782470703, 247.857666015625, 142.53195190429688, 101.13684844970703, 101.15641784667969, 80.84549713134766, 533.1124267578125, 500.5369873046875, 718.2415161132812]], "actual_norm": [[77.87776947021484, 92.651611328125, 23.144630432128906, 204.14418029785156, 193.6999969482422, 247.91610717773438, 142.51177978515625, 101.16242980957031, 101.1993408203125, 80.88822174072266, 533.14013671875, 500.48663330078125, 718.3472290039062]], "relative_norm": [[0.004924016073346138, 0.11837662011384964, 0.020033901557326317, 0.18035751581192017, 0.1821507215499878, 0.22973787784576416, 0.13758689165115356, 0.10167054831981659, 0.10529080033302307, 0.06347598135471344, 0.5796262621879578, 0.45526376366615295, 0.6979308128356934]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 1965, 10273, 34974, 431, 68825, 220, 19551, 8359, 6633], "first_top_logits": [29.375, 26.875, 22.5, 20.25, 20.0, 19.75, 19.625, 19.375, 19.375, 19.375], "source_correct": true, "target_correct": false, "seconds": 0.4653970832005143, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.25720253586769104, "target_logp": -35.86788558959961, "log_odds": -35.61068305373192, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.2309240698814392, -0.02620766870677471, -7.080780778778717e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-35.79342269897461, -0.07065960019826889, -0.003804827108979225]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.94572448730469, 83.613525390625, 25.089303970336914, 205.31903076171875, 195.20919799804688, 248.92747497558594, 144.9871826171875, 94.5771255493164, 94.01590728759766, 84.25273132324219, 549.2571411132812, 512.0056762695312, 752.4459228515625]], "relative_norm": [[0.0049283127300441265, 0.10526160150766373, 0.021707506850361824, 0.18085868656635284, 0.1838008463382721, 0.23024900257587433, 0.13963216543197632, 0.09534166008234024, 0.0980181023478508, 0.06638357043266296, 0.599038302898407, 0.46338850259780884, 0.727616548538208]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.9731750488281, 752.5794067382812]], "actual_norm": [[77.94572448730469, 83.613525390625, 25.089303970336914, 205.31903076171875, 195.20919799804688, 248.92747497558594, 144.9871826171875, 94.5771255493164, 94.01590728759766, 84.25273132324219, 549.2571411132812, 512.0056762695312, 752.4459228515625]], "relative_norm": [[0.0049283127300441265, 0.10526160150766373, 0.021707506850361824, 0.18085868656635284, 0.1838008463382721, 0.23024900257587433, 0.13963216543197632, 0.09534166008234024, 0.0980181023478508, 0.06638357043266296, 0.599038302898407, 0.46338850259780884, 0.727616548538208]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The country is Russia", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 3146, 374, 8359], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.6080551147461, 23.151418685913086, 204.158203125, 193.7395782470703, 247.857666015625, 142.53195190429688, 101.13684844970703, 101.15641784667969, 80.84549713134766, 533.1124267578125, 500.5370178222656, 718.2415161132812]], "actual_norm": [[77.94572448730469, 92.61573028564453, 23.221607208251953, 204.15074157714844, 193.80279541015625, 247.89768981933594, 142.51780700683594, 101.10648345947266, 101.15176391601562, 80.8355941772461, 533.1143798828125, 500.5810546875, 718.227783203125]], "relative_norm": [[0.0049283127300441265, 0.11833077669143677, 0.020100533962249756, 0.18036331236362457, 0.18224740028381348, 0.229720801115036, 0.13759270310401917, 0.1016143187880516, 0.10524129867553711, 0.06343468278646469, 0.5795982480049133, 0.4553496241569519, 0.6978147625923157]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 8359, 6364, 220, 435, 320, 1352, 3946, 104084, 22060], "first_top_logits": [26.25, 24.625, 19.0, 18.625, 18.625, 18.0, 17.75, 17.5, 17.25, 17.125], "source_correct": true, "target_correct": false, "seconds": 0.4493126249872148, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.0642775148153305, "target_logp": -13.151128768920898, "log_odds": -13.086851254105568, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.004890148062258959, -0.05933217331767082, -5.519237674889155e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-13.067390441894531, -0.0796104148030281, -0.004127318039536476]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17578125, 248.94473266601562, 145.0062255859375, 94.59593200683594, 94.04039764404297, 84.31642150878906, 549.3352661132812, 511.97314453125, 752.579345703125]], "actual_norm": [[77.74423217773438, 83.6429214477539, 25.0695858001709, 205.2458953857422, 195.16876220703125, 248.91018676757812, 144.94207763671875, 94.6701431274414, 94.01881408691406, 84.40753936767578, 549.4202880859375, 511.9667663574219, 752.6012573242188]], "relative_norm": [[0.004915573168545961, 0.10529860854148865, 0.021690446883440018, 0.18079426884651184, 0.18376277387142181, 0.23023301362991333, 0.13958872854709625, 0.09543542563915253, 0.09802113473415375, 0.0665055438876152, 0.5992162227630615, 0.46335330605506897, 0.727766752243042]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17578125, 248.94473266601562, 145.0062255859375, 94.59593200683594, 94.04039764404297, 84.31642150878906, 549.3352661132812, 511.97314453125, 752.579345703125]], "actual_norm": [[77.74423217773438, 83.6429214477539, 25.0695858001709, 205.2458953857422, 195.16876220703125, 248.91018676757812, 144.94207763671875, 94.6701431274414, 94.01881408691406, 84.40753936767578, 549.4202880859375, 511.9667663574219, 752.6012573242188]], "relative_norm": [[0.004915573168545961, 0.10529860854148865, 0.021690446883440018, 0.18079426884651184, 0.18376277387142181, 0.23023301362991333, 0.13958872854709625, 0.09543542563915253, 0.09802113473415375, 0.0665055438876152, 0.5992162227630615, 0.46335330605506897, 0.727766752243042]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  What would be the best way to", "output_ids": [8522, 10273, 273, 13, 220, 3555, 1035, 387, 279, 1850, 1616, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.60804748535156, 23.151418685913086, 204.158203125, 193.73956298828125, 247.85765075683594, 142.53195190429688, 101.1368408203125, 101.15641021728516, 80.84549713134766, 533.1124267578125, 500.5369873046875, 718.241455078125]], "actual_norm": [[77.74423217773438, 92.5739974975586, 23.251792907714844, 204.1464080810547, 193.65126037597656, 247.85044860839844, 142.55845642089844, 101.18028259277344, 101.16165924072266, 80.81464385986328, 533.1725463867188, 500.61627197265625, 718.2998046875]], "relative_norm": [[0.004915573168545961, 0.11827745288610458, 0.02012666128575802, 0.18035948276519775, 0.1821049004793167, 0.22967703640460968, 0.13763195276260376, 0.10168848931789398, 0.10525159537792206, 0.06341823935508728, 0.57966148853302, 0.4553816616535187, 0.697884738445282]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 10273, 19390, 431, 12839, 17672, 435, 19461, 33966, 4882], "first_top_logits": [27.625, 20.875, 20.625, 20.375, 20.25, 19.5, 18.5, 18.25, 18.0, 17.875], "source_correct": true, "target_correct": false, "seconds": 0.43280695797875524, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6214174032211304, "target_logp": -13.948838233947754, "log_odds": -13.327420830726624, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.5697100162506104, -0.05161558464169502, -9.178694017464295e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-13.882209777832031, -0.06325587630271912, -0.0033718657214194536]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.94476318359375, 145.0062255859375, 94.595947265625, 94.04041290283203, 84.3164291381836, 549.3352661132812, 511.97320556640625, 752.5794067382812]], "actual_norm": [[77.84272003173828, 83.68331909179688, 25.091106414794922, 205.3304901123047, 195.18360900878906, 248.93734741210938, 145.0399932861328, 94.56425476074219, 94.04302215576172, 84.4242172241211, 549.3441772460938, 511.9263000488281, 752.7051391601562]], "relative_norm": [[0.0049217999912798405, 0.10534946620464325, 0.02170906588435173, 0.18086878955364227, 0.18377675116062164, 0.23025813698768616, 0.13968303799629211, 0.0953286811709404, 0.09804636985063553, 0.06651867926120758, 0.5991332530975342, 0.4633166790008545, 0.7278671860694885]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.94476318359375, 145.0062255859375, 94.595947265625, 94.04041290283203, 84.3164291381836, 549.3352661132812, 511.97320556640625, 752.5794067382812]], "actual_norm": [[77.84272003173828, 83.68331909179688, 25.091106414794922, 205.3304901123047, 195.18360900878906, 248.93734741210938, 145.0399932861328, 94.56425476074219, 94.04302215576172, 84.4242172241211, 549.3441772460938, 511.9263000488281, 752.7051391601562]], "relative_norm": [[0.0049217999912798405, 0.10534946620464325, 0.02170906588435173, 0.18086878955364227, 0.18377675116062164, 0.23025813698768616, 0.13968303799629211, 0.0953286811709404, 0.09804636985063553, 0.06651867926120758, 0.5991332530975342, 0.4633166790008545, 0.7278671860694885]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  What would be the best way to", "output_ids": [8522, 10273, 273, 13, 220, 3555, 1035, 387, 279, 1850, 1616, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.60806274414062, 23.151418685913086, 204.158203125, 193.73959350585938, 247.85768127441406, 142.53195190429688, 101.13685607910156, 101.15641784667969, 80.84549713134766, 533.1124267578125, 500.5370178222656, 718.2415771484375]], "actual_norm": [[77.84272003173828, 92.60838317871094, 23.143802642822266, 204.1685028076172, 193.75169372558594, 247.796630859375, 142.57632446289062, 101.18524932861328, 101.17256164550781, 80.85428619384766, 533.0982666015625, 500.5889587402344, 718.1812744140625]], "relative_norm": [[0.0049217999912798405, 0.11832138895988464, 0.020033186301589012, 0.1803790032863617, 0.18219934403896332, 0.22962716221809387, 0.13764919340610504, 0.10169348120689392, 0.10526293516159058, 0.06344935297966003, 0.5795807242393494, 0.4553568363189697, 0.6977695822715759]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 10273, 431, 435, 12839, 13, 547, 34974, 5429, 94705], "first_top_logits": [21.875, 20.875, 19.125, 18.25, 17.5, 17.0, 16.75, 16.375, 16.25, 16.125], "source_correct": true, "target_correct": false, "seconds": 0.44086966710165143, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-11/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-11", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Kazan", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Kazan is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.07971616834402084, "target_logp": -16.939828872680664, "log_odds": -16.860112704336643, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.006367513909935951, -0.0732818990945816, -6.675497570540756e-05]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-16.881366729736328, -0.05546530708670616, -0.0029962430708110332]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17576599121094, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.97314453125, 752.5794067382812]], "actual_norm": [[77.77253723144531, 83.60671997070312, 25.145719528198242, 205.39967346191406, 195.11180114746094, 248.93984985351562, 145.03175354003906, 94.51805114746094, 94.06295776367188, 84.33454895019531, 549.32080078125, 511.90802001953125, 752.6787719726562]], "relative_norm": [[0.004917362704873085, 0.10525304079055786, 0.02175631746649742, 0.18092972040176392, 0.18370914459228516, 0.2302604466676712, 0.1396750956773758, 0.09528210759162903, 0.09806715697050095, 0.06644803285598755, 0.5991077423095703, 0.4633001387119293, 0.7278417348861694]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17576599121094, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 84.3164291381836, 549.3352661132812, 511.97314453125, 752.5794067382812]], "actual_norm": [[77.77253723144531, 83.60671997070312, 25.145719528198242, 205.39967346191406, 195.11180114746094, 248.93984985351562, 145.03175354003906, 94.51805114746094, 94.06295776367188, 84.33454895019531, 549.32080078125, 511.90802001953125, 752.6787719726562]], "relative_norm": [[0.004917362704873085, 0.10525304079055786, 0.02175631746649742, 0.18092972040176392, 0.18370914459228516, 0.2302604466676712, 0.1396750956773758, 0.09528210759162903, 0.09806715697050095, 0.06644803285598755, 0.5991077423095703, 0.4633001387119293, 0.7278417348861694]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The country is Russia", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 3146, 374, 8359], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 34974, 276, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[77.88261413574219, 92.6080551147461, 23.151418685913086, 204.158203125, 193.73956298828125, 247.85765075683594, 142.53195190429688, 101.1368408203125, 101.15641784667969, 80.84548950195312, 533.1123657226562, 500.5369873046875, 718.241455078125]], "actual_norm": [[77.77253723144531, 92.59408569335938, 23.236528396606445, 204.22708129882812, 193.75694274902344, 247.87161254882812, 142.53390502929688, 101.14781951904297, 101.12591552734375, 80.87252807617188, 533.0476684570312, 500.5295104980469, 718.3014526367188]], "relative_norm": [[0.004917362704873085, 0.11830312013626099, 0.020113449543714523, 0.18043075501918793, 0.18220427632331848, 0.22969664633274078, 0.13760824501514435, 0.10165586322546005, 0.10521440207958221, 0.06346366554498672, 0.5795257091522217, 0.45530274510383606, 0.6978863477706909]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 431, 809, 19390, 10273, 4882, 435, 12839, 68825, 8359], "first_top_logits": [28.5, 22.625, 21.875, 21.375, 20.125, 19.75, 19.375, 18.5, 18.375, 18.375], "source_correct": true, "target_correct": false, "seconds": 0.45807037502527237, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/readout", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 176.0, "target_rank": 541.0, "source_logit": 4.875, "target_logit": 3.578125, "top_ids": [11413, 45418, 15973, 8330, 38352, 6364, 7513, 3693, 28453, 49029, 11888, 90475], "top_tokens": [" currency", ".currency", "currency", " Australia", "uenta", " English", " European", " American", " Currency", "ctica", " Canadian", "(currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 166.5, "target_rank": 4493.0, "source_logit": 5.3125, "target_logit": 1.046875, "top_ids": [11413, 15973, 96694, 3693, 28453, 3001, 18174, 45418, 5088, 7065, 53661, 126256], "top_tokens": [" currency", "currency", "onis", " American", " Currency", "ica", "oret", ".currency", "urrency", "ican", "cla", "a\u0142"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 23.0, "target_rank": 484.0, "source_logit": 22.0, "target_logit": 16.0, "top_ids": [11413, 35602, 90475, 32488, 7214, 7513, 28453, 77186, 6489, 45418, 36445, 85466], "top_tokens": [" currency", " currencies", "(currency", " euros", " foreign", " European", " Currency", ".Currency", " international", ".currency", " internationally", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 34.5, "target_rank": 1195.5, "source_logit": 17.125, "target_logit": 11.5625, "top_ids": [11413, 35602, 7214, 47412, 3220, 65001, 15973, 34710, 21130, 78253, 32284, 17672], "top_tokens": [" currency", " currencies", " foreign", " foreigners", " money", "\"\u2014", "currency", "_currency", " parliament", " treasury", " monetary", " euro"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 781.0, "target_rank": 2890.5, "source_logit": 2.984375, "target_logit": 1.3984375, "top_ids": [21251, 28453, 3776, 7748, 11413, 21525, 3731, 55964, 4229, 5737, 5364, 22375], "top_tokens": ["onia", " Currency", " One", " Indian", " currency", " Ring", " Red", "\u2011", " Ge", " Group", "ador", " Atlantic"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 490.0, "target_rank": 1804.0, "source_logit": 3.375, "target_logit": 1.7265625, "top_ids": [11413, 18174, 479, 7179, 11766, 364, 21251, 2932, 362, 4102, 28453, 4716], "top_tokens": [" currency", "oret", " G", " International", " Standard", " '", "onia", " She", " A", "\u00a0", " Currency", " Bar"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 842.5, "target_rank": 1017.5, "source_logit": 14.5, "target_logit": 14.1875, "top_ids": [11413, 35602, 28453, 5837, 7576, 5313, 3146, 43895, 7214, 32488, 13867, 29970], "top_tokens": [" currency", " currencies", " Currency", " countries", " prices", " national", " country", " tariffs", " foreign", " euros", " county", " tourist"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 95.5, "target_rank": 943.5, "source_logit": 15.0625, "target_logit": 12.125, "top_ids": [11413, 35602, 37198, 7214, 6489, 5313, 105492, 5837, 94936, 86049, 7513, 17672], "top_tokens": [" currency", " currencies", " Commonwealth", " foreign", " international", " national", "\u4e2d\u534e\u4eba\u6c11", " countries", " denomination", " monarchy", " European", " euro"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 2971.0, "target_rank": 4564.0, "source_logit": 1.1796875, "target_logit": 0.50390625, "top_ids": [3693, 4787, 89478, 279, 5434, 21251, 15503, 4229, 6364, 111012, 2240, 1083], "top_tokens": [" American", " North", "\u2026\n", " the", "\u2026\n\n", "onia", " primarily", " Ge", " English", "\u4e16\u754c\u4e0a\u6700", " None", " also"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 841.0, "target_rank": 3810.0, "source_logit": 2.796875, "target_logit": 0.6328125, "top_ids": [1852, 5938, 3639, 3693, 4102, 5055, 18174, 271, 4787, 479, 5313, 2701], "top_tokens": [" same", " German", " United", " American", "\u00a0", " National", "oret", "\n\n", " North", " G", " national", " following"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 969.0, "target_rank": 5470.5, "source_logit": 7.875, "target_logit": 4.375, "top_ids": [104142, 103408, 113883, 35602, 11413, 115160, 106333, 90475, 116588, 101237, 28453, 26321], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u672c\u56fd", " currencies", " currency", "\u5176\u4ed6\u56fd\u5bb6", "\u5916\u6c47", "(currency", "\u6210\u5458\u56fd", "\u7f8e\u5143", " Currency", "Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 842.5, "target_rank": 14428.5, "source_logit": 6.53125, "target_logit": 0.84765625, "top_ids": [104142, 116866, 112397, 109442, 99599, 113883, 103408, 106333, 101237, 116588, 104773, 102120], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u8d22\u653f\u90e8", "\u56fd\u5bb6", "\u672c\u56fd", "\u6b27\u5143", "\u5916\u6c47", "\u7f8e\u5143", "\u6210\u5458\u56fd", "\u5171\u548c\u56fd", "\u8d22\u653f"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 15.0, "target_rank": 12541.5, "source_logit": 18.375, "target_logit": 0.65234375, "top_ids": [8522, 11413, 11192, 104142, 17692, 19551, 2598, 28453, 47707, 35602, 104084, 33625], "top_tokens": [" Russian", " currency", " dollars", "\u4eba\u6c11\u5e01", " dollar", " USD", " called", " Currency", "Russian", " currencies", "\u4fc4\u7f57\u65af", " Ukrainian"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 7.5, "target_rank": 5476.5, "source_logit": 20.75, "target_logit": 1.65625, "top_ids": [8522, 11413, 104084, 19390, 104142, 33625, 8359, 19551, 47707, 17692, 63145, 22415], "top_tokens": [" Russian", " currency", "\u4fc4\u7f57\u65af", " Soviet", "\u4eba\u6c11\u5e01", " Ukrainian", " Russia", " USD", "Russian", " dollar", " russian", " Moscow"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 32.0, "target_rank": 7095.0, "source_logit": 25.5, "target_logit": 4.59375, "top_ids": [104142, 11413, 11192, 35602, 101406, 28453, 17692, 101237, 103408, 32488, 45418, 80189], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " dollars", " currencies", "\u8d27\u5e01", " Currency", " dollar", "\u7f8e\u5143", "\u6b27\u5143", " euros", ".currency", " Dollars"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 16.0, "target_rank": 7329.0, "source_logit": 22.375, "target_logit": 2.09375, "top_ids": [104142, 8522, 11413, 104084, 101406, 35602, 101237, 45418, 19551, 11192, 17692, 33625], "top_tokens": ["\u4eba\u6c11\u5e01", " Russian", " currency", "\u4fc4\u7f57\u65af", "\u8d27\u5e01", " currencies", "\u7f8e\u5143", ".currency", " USD", " dollars", " dollar", " Ukrainian"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 36.0, "target_rank": 70591.0, "source_logit": 17.0, "target_logit": -2.671875, "top_ids": [279, 10273, 8522, 2598, 3881, 12839, 2176, 5644, 220, 2987, 305, 1083], "top_tokens": [" the", " rub", " Russian", " called", " known", " Rub", " both", " held", " ", " either", " h", " also"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 9.0, "target_rank": 51010.0, "source_logit": 19.625, "target_logit": -1.7734375, "top_ids": [8522, 10273, 12839, 94705, 19390, 435, 595, 431, 8359, 104084, 4741, 63145], "top_tokens": [" Russian", " rub", " Rub", " Cyril", " Soviet", " r", " k", " R", " Russia", "\u4fc4\u7f57\u65af", " former", " russian"]}], "source_token": 8359, "target_token": 17009, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Nov", "os", "ibir", "sk", " is", " the"], "seconds": 1.144218291155994}
{"run_id": "eval-12/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.04838180914521217, "target_logp": -23.841182708740234, "log_odds": -23.792800899595022, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.02340092323720455, -0.024880632758140564, -0.00010024998482549563]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-23.710901260375977, -0.12740640342235565, -0.002873579505831003]}}, "score_trace": {"source": [], "target": []}, "output": " Russian ruble.  Inference: The currency used in", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [], "first_top_ids": [8522, 10273, 12839, 94705, 19390, 435, 595, 431, 8359, 104084], "first_top_logits": [29.5, 25.625, 23.25, 21.25, 21.125, 20.75, 20.5, 20.375, 19.625, 19.0], "source_correct": true, "target_correct": false, "seconds": 0.5118084163405001, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.04838180914521217, "target_logp": -23.841182708740234, "log_odds": -23.792800899595022, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.02340092323720455, -0.024880632758140564, -0.00010024998482549563]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-23.710901260375977, -0.12740640342235565, -0.002873579505831003]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The currency used in", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 10273, 12839, 94705, 19390, 435, 595, 431, 8359, 104084], "first_top_logits": [29.5, 25.625, 23.25, 21.25, 21.125, 20.75, 20.5, 20.375, 19.625, 19.0], "source_correct": true, "target_correct": false, "seconds": 0.48480533296242356, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -46.64437484741211, "target_logp": -6.958447456359863, "log_odds": 39.685927391052246, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-46.563690185546875, -0.08051668107509613, -0.0001683093432802707]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-6.7511887550354, -0.201689213514328, -0.005569654516875744]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0440063476562, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.9727783203125, 83.60452270507812, 25.083982467651367, 205.33782958984375, 195.2326202392578, 248.97811889648438, 145.00331115722656, 94.5683364868164, 94.02424621582031, 166.00021362304688, 624.1175537109375, 663.9544677734375, 822.022705078125, 1010.0755004882812, 1032.1514892578125]], "relative_norm": [[0.004930023569613695, 0.1052502691745758, 0.021702902391552925, 0.1808752417564392, 0.183822900056839, 0.23029585182666779, 0.1396477073431015, 0.0953327938914299, 0.09802679717540741, 0.13723711669445038, 0.5058805346488953, 0.7219062447547913, 0.8902842998504639, 0.8990070819854736, 0.9652419090270996]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0440063476562, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.9727783203125, 83.60452270507812, 25.083982467651367, 205.33782958984375, 195.2326202392578, 248.97811889648438, 145.00331115722656, 94.5683364868164, 94.02424621582031, 166.00021362304688, 624.1175537109375, 663.9544677734375, 822.022705078125, 1010.0755004882812, 1032.1514892578125]], "relative_norm": [[0.004930023569613695, 0.1052502691745758, 0.021702902391552925, 0.1808752417564392, 0.183822900056839, 0.23029585182666779, 0.1396477073431015, 0.0953327938914299, 0.09802679717540741, 0.13723711669445038, 0.5058805346488953, 0.7219062447547913, 0.8902842998504639, 0.8990070819854736, 0.9652419090270996]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkeylite.  What is the currency of the country containing", "output_ids": [17009, 68078, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390991210938, 25.782773971557617, 207.01571655273438, 194.4975128173828, 247.47190856933594, 145.3911590576172, 98.75170135498047, 99.13423919677734, 165.24609375, 623.1695556640625, 655.061279296875, 813.3048095703125, 989.63671875, 1026.634765625]], "actual_norm": [[77.9727783203125, 85.41963195800781, 25.823579788208008, 207.0569305419922, 194.54153442382812, 247.4107666015625, 145.38656616210938, 98.79190063476562, 99.14318084716797, 165.26242065429688, 623.2167358398438, 655.0982666015625, 813.3068237304688, 989.64794921875, 1026.7012939453125]], "relative_norm": [[0.004930023569613695, 0.10780549049377441, 0.022292660549283028, 0.18260733783245087, 0.18330375850200653, 0.22886331379413605, 0.14013637602329254, 0.09954230487346649, 0.1030954048037529, 0.13647307455539703, 0.5036997199058533, 0.7127729654312134, 0.8820884823799133, 0.8835455775260925, 0.9643658399581909]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17009, 23734, 44305, 350, 72719, 109236, 17860, 42364, 69010, 1532], "first_top_logits": [31.75, 25.0, 19.875, 19.625, 19.375, 17.0, 16.5, 15.5625, 15.5625, 15.3125], "source_correct": false, "target_correct": false, "seconds": 0.4255530000664294, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.5957210063934326, "target_logp": -21.342082977294922, "log_odds": -18.74636197090149, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-2.581212043762207, -0.014272631146013737, -0.00023636408150196075]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-21.23746109008789, -0.10058082640171051, -0.004041718319058418]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.595947265625, 94.04041290283203, 165.93112182617188, 624.0439453125, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.87776947021484, 83.6090087890625, 25.066511154174805, 205.3695068359375, 195.1577911376953, 248.9435272216797, 145.02371215820312, 94.62918853759766, 94.04428100585938, 165.8930206298828, 624.1134643554688, 663.9572143554688, 822.0267333984375, 1010.1384887695312, 1032.1527099609375]], "relative_norm": [[0.004924016073346138, 0.10525591671466827, 0.021687785163521767, 0.18090315163135529, 0.18375244736671448, 0.2302638441324234, 0.139667347073555, 0.09539414197206497, 0.09804768115282059, 0.13714848458766937, 0.5058771967887878, 0.72190922498703, 0.8902887105941772, 0.8990631699562073, 0.9652430415153503]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.595947265625, 94.04041290283203, 165.93112182617188, 624.0439453125, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.87776947021484, 83.6090087890625, 25.066511154174805, 205.3695068359375, 195.1577911376953, 248.9435272216797, 145.02371215820312, 94.62918853759766, 94.04428100585938, 165.8930206298828, 624.1134643554688, 663.9572143554688, 822.0267333984375, 1010.1384887695312, 1032.1527099609375]], "relative_norm": [[0.004924016073346138, 0.10525591671466827, 0.021687785163521767, 0.18090315163135529, 0.18375244736671448, 0.2302638441324234, 0.139667347073555, 0.09539414197206497, 0.09804768115282059, 0.13714848458766937, 0.5058771967887878, 0.72190922498703, 0.8902887105941772, 0.8990631699562073, 0.9652430415153503]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Nov", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 4620], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390991210938, 25.782772064208984, 207.01571655273438, 194.4975128173828, 247.47190856933594, 145.3911590576172, 98.75170135498047, 99.13423156738281, 165.24609375, 623.1695556640625, 655.061279296875, 813.3048095703125, 989.63671875, 1026.634765625]], "actual_norm": [[77.87776947021484, 85.40824127197266, 25.7723445892334, 207.08346557617188, 194.52638244628906, 247.49749755859375, 145.39781188964844, 98.75139617919922, 99.18096160888672, 165.28929138183594, 623.3399658203125, 655.1326293945312, 813.3585205078125, 989.664306640625, 1026.5418701171875]], "relative_norm": [[0.004924016073346138, 0.10779111832380295, 0.02224843204021454, 0.1826307475566864, 0.18328946828842163, 0.22894354164600372, 0.14014722406864166, 0.09950149804353714, 0.10313469171524048, 0.13649527728557587, 0.503799319267273, 0.7128103375434875, 0.8821445107460022, 0.8835601806640625, 0.9642161130905151]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 8522, 6633, 5313, 220, 10273, 198, 320, 19390, 1140], "first_top_logits": [28.5, 26.0, 21.125, 20.75, 20.25, 19.875, 19.5, 19.25, 19.0, 18.5], "source_correct": false, "target_correct": false, "seconds": 0.49880237504839897, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.4054999351501465, "target_logp": -38.25907516479492, "log_odds": -36.853575229644775, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-1.392090916633606, -0.013264154084026814, -0.00014482879487331957]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-38.14208984375, -0.11329451948404312, -0.0036925957538187504]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0440063476562, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.94572448730469, 83.613525390625, 25.089303970336914, 205.31903076171875, 195.20919799804688, 248.92747497558594, 144.9871826171875, 94.5771255493164, 94.01590728759766, 165.8921661376953, 624.1023559570312, 663.950439453125, 822.064453125, 1010.0226440429688, 1032.140869140625]], "relative_norm": [[0.0049283127300441265, 0.10526160150766373, 0.021707506850361824, 0.18085868656635284, 0.1838008463382721, 0.23024900257587433, 0.13963216543197632, 0.09534166008234024, 0.0980181023478508, 0.13714778423309326, 0.5058681964874268, 0.7219018936157227, 0.8903295397758484, 0.8989600539207458, 0.9652319550514221]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0440063476562, 664.0018920898438, 822.0299072265625, 1010.072265625, 1032.1513671875]], "actual_norm": [[77.94572448730469, 83.613525390625, 25.089303970336914, 205.31903076171875, 195.20919799804688, 248.92747497558594, 144.9871826171875, 94.5771255493164, 94.01590728759766, 165.8921661376953, 624.1023559570312, 663.950439453125, 822.064453125, 1010.0226440429688, 1032.140869140625]], "relative_norm": [[0.0049283127300441265, 0.10526160150766373, 0.021707506850361824, 0.18085868656635284, 0.1838008463382721, 0.23024900257587433, 0.13963216543197632, 0.09534166008234024, 0.0980181023478508, 0.13714778423309326, 0.5058681964874268, 0.7219018936157227, 0.8903295397758484, 0.8989600539207458, 0.9652319550514221]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russia ruble.\n\nFact: The currency used in the country", "output_ids": [8359, 10273, 273, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390991210938, 25.782773971557617, 207.01571655273438, 194.4975128173828, 247.47190856933594, 145.3911590576172, 98.75170135498047, 99.13423919677734, 165.24609375, 623.1695556640625, 655.061279296875, 813.3048095703125, 989.63671875, 1026.634765625]], "actual_norm": [[77.94572448730469, 85.41895294189453, 25.819639205932617, 206.93345642089844, 194.57102966308594, 247.52105712890625, 145.37762451171875, 98.75819396972656, 99.15702819824219, 165.29891967773438, 623.3335571289062, 655.0269775390625, 813.2024536132812, 989.585693359375, 1026.612060546875]], "relative_norm": [[0.0049283127300441265, 0.10780463367700577, 0.022289259359240532, 0.18249844014644623, 0.18333154916763306, 0.22896532714366913, 0.1401277631521225, 0.09950834512710571, 0.10310979932546616, 0.1365032196044922, 0.5037941336631775, 0.7126954197883606, 0.8819752335548401, 0.8834900259971619, 0.9642820358276367]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8359, 8522, 435, 220, 6364, 4741, 320, 505, 10273, 773], "first_top_logits": [25.375, 24.5, 22.75, 19.75, 19.625, 19.25, 18.875, 18.625, 18.5, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.4497506250627339, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.060017213225364685, "target_logp": -17.228437423706055, "log_odds": -17.16842021048069, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.034874893724918365, -0.025030389428138733, -0.00011193125828867778]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-17.097373962402344, -0.1274227797985077, -0.003641167189925909]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17578125, 248.94473266601562, 145.0062255859375, 94.59593200683594, 94.04039764404297, 165.93112182617188, 624.0439453125, 664.0018920898438, 822.0299072265625, 1010.0722045898438, 1032.1513671875]], "actual_norm": [[77.74423217773438, 83.6429214477539, 25.0695858001709, 205.2458953857422, 195.16876220703125, 248.91018676757812, 144.94207763671875, 94.6701431274414, 94.01881408691406, 165.97091674804688, 624.0903930664062, 663.9741821289062, 821.9609985351562, 1010.2357177734375, 1032.154296875]], "relative_norm": [[0.004915573168545961, 0.10529860854148865, 0.021690446883440018, 0.18079426884651184, 0.18376277387142181, 0.23023301362991333, 0.13958872854709625, 0.09543542563915253, 0.09802113473415375, 0.13721288740634918, 0.5058584809303284, 0.7219277024269104, 0.8902174830436707, 0.8991496562957764, 0.9652445316314697]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17578125, 248.94473266601562, 145.0062255859375, 94.59593200683594, 94.04039764404297, 165.93112182617188, 624.0439453125, 664.0018920898438, 822.0299072265625, 1010.0722045898438, 1032.1513671875]], "actual_norm": [[77.74423217773438, 83.6429214477539, 25.0695858001709, 205.2458953857422, 195.16876220703125, 248.91018676757812, 144.94207763671875, 94.6701431274414, 94.01881408691406, 165.97091674804688, 624.0903930664062, 663.9741821289062, 821.9609985351562, 1010.2357177734375, 1032.154296875]], "relative_norm": [[0.004915573168545961, 0.10529860854148865, 0.021690446883440018, 0.18079426884651184, 0.18376277387142181, 0.23023301362991333, 0.13958872854709625, 0.09543542563915253, 0.09802113473415375, 0.13721288740634918, 0.5058584809303284, 0.7219277024269104, 0.8902174830436707, 0.8991496562957764, 0.9652445316314697]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  What is the currency used in the", "output_ids": [8522, 10273, 273, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390228271484, 25.782772064208984, 207.01571655273438, 194.49749755859375, 247.47189331054688, 145.39114379882812, 98.75169372558594, 99.13423156738281, 165.24609375, 623.1694946289062, 655.0612182617188, 813.3047485351562, 989.6366577148438, 1026.634765625]], "actual_norm": [[77.74423217773438, 85.43386840820312, 25.843130111694336, 206.96725463867188, 194.4714813232422, 247.45741271972656, 145.3908233642578, 98.768310546875, 99.13862609863281, 165.32843017578125, 623.0897827148438, 655.0159912109375, 813.3037109375, 989.7178344726562, 1026.662109375]], "relative_norm": [[0.004915573168545961, 0.1078234612941742, 0.02230953797698021, 0.18252825736999512, 0.18323774635791779, 0.22890646755695343, 0.14014048874378204, 0.0995185375213623, 0.10309066623449326, 0.13652759790420532, 0.50359708070755, 0.7126834392547607, 0.8820850849151611, 0.8836079835891724, 0.9643290638923645]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 19390, 12839, 10273, 17672, 431, 19461, 220, 33966, 30], "first_top_logits": [26.5, 22.75, 21.625, 21.125, 20.375, 19.0, 18.5, 18.0, 17.375, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.45849716709926724, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9124356508255005, "target_logp": -16.101125717163086, "log_odds": -14.188690066337585, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-1.8918516635894775, -0.020420553162693977, -0.00016342257731594145]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-15.954351425170898, -0.14380240440368652, -0.002972234506160021]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.94476318359375, 145.0062255859375, 94.595947265625, 94.04041290283203, 165.93113708496094, 624.0440063476562, 664.001953125, 822.0299072265625, 1010.0723266601562, 1032.1514892578125]], "actual_norm": [[77.84272003173828, 83.68331909179688, 25.091106414794922, 205.3304901123047, 195.18360900878906, 248.93734741210938, 145.0399932861328, 94.56425476074219, 94.04302215576172, 165.97193908691406, 624.119384765625, 663.991455078125, 821.9359130859375, 1010.0784912109375, 1032.205322265625]], "relative_norm": [[0.0049217999912798405, 0.10534946620464325, 0.02170906588435173, 0.18086878955364227, 0.18377675116062164, 0.23025813698768616, 0.13968303799629211, 0.0953286811709404, 0.09804636985063553, 0.13721373677253723, 0.5058820247650146, 0.7219464778900146, 0.8901903033256531, 0.8990097641944885, 0.9652922749519348]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048709869384766, 205.3473358154297, 195.17578125, 248.94476318359375, 145.0062255859375, 94.595947265625, 94.04041290283203, 165.93113708496094, 624.0440063476562, 664.001953125, 822.0299072265625, 1010.0723266601562, 1032.1514892578125]], "actual_norm": [[77.84272003173828, 83.68331909179688, 25.091106414794922, 205.3304901123047, 195.18360900878906, 248.93734741210938, 145.0399932861328, 94.56425476074219, 94.04302215576172, 165.97193908691406, 624.119384765625, 663.991455078125, 821.9359130859375, 1010.0784912109375, 1032.205322265625]], "relative_norm": [[0.0049217999912798405, 0.10534946620464325, 0.02170906588435173, 0.18086878955364227, 0.18377675116062164, 0.23025813698768616, 0.13968303799629211, 0.0953286811709404, 0.09804636985063553, 0.13721373677253723, 0.5058820247650146, 0.7219464778900146, 0.8901903033256531, 0.8990097641944885, 0.9652922749519348]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " ruble.  What is the currency used in the country", "output_ids": [10273, 273, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390991210938, 25.782773971557617, 207.01571655273438, 194.49752807617188, 247.471923828125, 145.39117431640625, 98.751708984375, 99.13423919677734, 165.24609375, 623.1695556640625, 655.061279296875, 813.3048095703125, 989.63671875, 1026.634765625]], "actual_norm": [[77.84272003173828, 85.433349609375, 25.821765899658203, 207.0120391845703, 194.45936584472656, 247.44851684570312, 145.41090393066406, 98.78726196289062, 99.10987091064453, 165.21224975585938, 623.1928100585938, 655.0684814453125, 813.2489013671875, 989.5950317382812, 1026.7008056640625]], "relative_norm": [[0.0049217999912798405, 0.10782280564308167, 0.022291095927357674, 0.1825677454471588, 0.1832263320684433, 0.22889822721481323, 0.14015983045101166, 0.09953763335943222, 0.1030607670545578, 0.13643164932727814, 0.5036803483963013, 0.7127405405044556, 0.8820256590843201, 0.8834983110427856, 0.9643653631210327]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10273, 8522, 94705, 547, 892, 435, 13, 72437, 294, 34974], "first_top_logits": [19.125, 17.875, 16.625, 16.375, 15.875, 15.625, 15.5625, 15.125, 15.0625, 14.75], "source_correct": true, "target_correct": false, "seconds": 0.4477627919986844, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-12/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-12", "split": "eval", "country": "Russia", "currency": "Russian ruble", "aliases": ["ruble", "RUB", "rouble"], "city": "Novosibirsk", "target_country": "Turkey", "target_currency": "Turkish lira", "target_aliases": ["lira", "TRY"], "prompt": "Fact: The currency used in the country containing Novosibirsk is the", "source_continuation": " Russian ruble", "target_continuation": " Turkish lira"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.09168952703475952, "target_logp": -21.311260223388672, "log_odds": -21.219570696353912, "token_scores": {"source": {"ids": [8522, 10273, 273], "log_probs": [-0.07212018966674805, -0.019440123811364174, -0.0001292145170737058]}, "target": {"ids": [23734, 326, 8832], "log_probs": [-21.228370666503906, -0.07924287766218185, -0.003646393306553364]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17576599121094, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0439453125, 664.0018310546875, 822.0299072265625, 1010.0722045898438, 1032.1513671875]], "actual_norm": [[77.77253723144531, 83.60671997070312, 25.145719528198242, 205.39967346191406, 195.11180114746094, 248.93984985351562, 145.03175354003906, 94.51805114746094, 94.06295776367188, 165.97166442871094, 624.0614624023438, 664.008544921875, 822.0818481445312, 1010.0863037109375, 1032.2137451171875]], "relative_norm": [[0.004917362704873085, 0.10525304079055786, 0.02175631746649742, 0.18092972040176392, 0.18370914459228516, 0.2302604466676712, 0.1396750956773758, 0.09528210759162903, 0.09806715697050095, 0.13721351325511932, 0.5058350563049316, 0.7219650745391846, 0.8903483748435974, 0.8990166783332825, 0.9653001427650452]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 83.6269302368164, 25.048707962036133, 205.34732055664062, 195.17576599121094, 248.9447479248047, 145.0062255859375, 94.59593963623047, 94.0404052734375, 165.93112182617188, 624.0439453125, 664.0018310546875, 822.0299072265625, 1010.0722045898438, 1032.1513671875]], "actual_norm": [[77.77253723144531, 83.60671997070312, 25.145719528198242, 205.39967346191406, 195.11180114746094, 248.93984985351562, 145.03175354003906, 94.51805114746094, 94.06295776367188, 165.97166442871094, 624.0614624023438, 664.008544921875, 822.0818481445312, 1010.0863037109375, 1032.2137451171875]], "relative_norm": [[0.004917362704873085, 0.10525304079055786, 0.02175631746649742, 0.18092972040176392, 0.18370914459228516, 0.2302604466676712, 0.1396750956773758, 0.09528210759162903, 0.09806715697050095, 0.13721351325511932, 0.5058350563049316, 0.7219650745391846, 0.8903483748435974, 0.8990166783332825, 0.9653001427650452]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Russian ruble.  Inference: The currency used in", "output_ids": [8522, 10273, 273, 13, 220, 758, 2202, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 4620, 436, 82460, 4886, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[77.88261413574219, 85.41390228271484, 25.782773971557617, 207.01571655273438, 194.4975128173828, 247.47189331054688, 145.3911590576172, 98.75169372558594, 99.13423919677734, 165.24609375, 623.1694946289062, 655.061279296875, 813.3048095703125, 989.6366577148438, 1026.634765625]], "actual_norm": [[77.77253723144531, 85.44357299804688, 25.849205017089844, 207.10580444335938, 194.46456909179688, 247.52479553222656, 145.3546142578125, 98.78264617919922, 99.14188385009766, 165.29774475097656, 623.1923217773438, 655.0721435546875, 813.221923828125, 989.6378784179688, 1026.791259765625]], "relative_norm": [[0.004917362704873085, 0.10783570259809494, 0.022314783185720444, 0.18265044689178467, 0.18323123455047607, 0.2289687991142273, 0.14010557532310486, 0.09953298419713974, 0.10309405624866486, 0.1365022510290146, 0.5036799907684326, 0.7127445340156555, 0.8819963932037354, 0.8835365772247314, 0.9644503593444824]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [8522, 19390, 809, 67500, 431, 19551, 8359, 26, 3639, 10882], "first_top_logits": [25.75, 23.25, 18.125, 17.5, 17.5, 17.375, 16.875, 16.625, 16.125, 16.0], "source_correct": true, "target_correct": false, "seconds": 0.47480300022289157, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/readout", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 625.0, "target_rank": 1841.5, "source_logit": 3.359375, "target_logit": 2.0, "top_ids": [11413, 8330, 38352, 15973, 45418, 3693, 6364, 31491, 7065, 7513, 7748, 28453], "top_tokens": [" currency", " Australia", "uenta", "currency", ".currency", " American", " English", "uesto", "ican", " European", " Indian", " Currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 2489.0, "target_rank": 2162.0, "source_logit": 1.5390625, "target_logit": 1.734375, "top_ids": [11413, 15973, 96694, 3001, 3693, 28453, 53661, 7065, 6364, 5088, 45418, 34065], "top_tokens": [" currency", "currency", "onis", "ica", " American", " Currency", "cla", "ican", " English", "urrency", ".currency", "_money"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 259.5, "target_rank": 395.0, "source_logit": 17.0, "target_logit": 16.0, "top_ids": [11413, 35602, 32488, 90475, 85466, 7513, 6489, 45418, 77186, 24357, 8330, 7214], "top_tokens": [" currency", " currencies", " euros", "(currency", " Euros", " European", " international", ".currency", ".Currency", " overseas", " Australia", " foreign"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 405.5, "target_rank": 583.0, "source_logit": 13.1875, "target_logit": 12.5625, "top_ids": [11413, 7214, 47412, 35602, 3220, 7855, 13224, 6489, 78253, 17672, 21130, 32488], "top_tokens": [" currency", " foreign", " foreigners", " currencies", " money", " British", " Britain", " international", " treasury", " euro", " parliament", " euros"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 5879.0, "target_rank": 3471.0, "source_logit": 0.228515625, "target_logit": 1.0, "top_ids": [21251, 28453, 7748, 12751, 55964, 3776, 3731, 603, 11413, 7513, 29921, 19833], "top_tokens": ["onia", " Currency", " Indian", " Roman", "\u2011", " One", " Red", "one", " currency", " European", "\u2010", " Amb"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 2257.0, "target_rank": 5421.5, "source_logit": 1.2890625, "target_logit": -0.05908203125, "top_ids": [364, 7179, 11413, 479, 11766, 467, 2932, 18174, 547, 356, 28453, 362], "top_tokens": [" '", " International", " currency", " G", " Standard", " W", " She", "oret", " U", " C", " Currency", " A"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 814.5, "target_rank": 2036.0, "source_logit": 14.25, "target_logit": 12.6875, "top_ids": [11413, 35602, 13867, 3146, 28453, 5313, 30176, 60261, 5837, 7576, 43895, 32488], "top_tokens": [" currency", " currencies", " county", " country", " Currency", " national", " counties", " tariff", " countries", " prices", " tariffs", " euros"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 764.5, "target_rank": 373.5, "source_logit": 12.8125, "target_logit": 13.875, "top_ids": [11413, 35602, 7513, 7214, 5837, 37198, 94936, 105492, 6489, 6364, 17672, 7855], "top_tokens": [" currency", " currencies", " European", " foreign", " countries", " Commonwealth", " denomination", "\u4e2d\u534e\u4eba\u6c11", " international", " English", " euro", " British"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 5500.0, "target_rank": 6617.0, "source_logit": 0.0078125, "target_logit": -0.2734375, "top_ids": [3693, 6364, 4787, 4229, 89478, 279, 21251, 7855, 3639, 15503, 5434, 2598], "top_tokens": [" American", " English", " North", " Ge", "\u2026\n", " the", "onia", " British", " United", " primarily", "\u2026\n\n", " called"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 2956.5, "target_rank": 5668.0, "source_logit": 0.78515625, "target_logit": -0.2060546875, "top_ids": [1852, 5938, 3693, 4102, 4787, 3639, 4882, 6364, 271, 479, 2274, 7179], "top_tokens": [" same", " German", " American", "\u00a0", " North", " United", " South", " English", "\n\n", " G", " US", " International"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1925.0, "target_rank": 1925.0, "source_logit": 9.4375, "target_logit": 9.4375, "top_ids": [11413, 104142, 103408, 35602, 28453, 113883, 32488, 26321, 106333, 90475, 112455, 85466], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " currencies", " Currency", "\u672c\u56fd", " euros", "Currency", "\u5916\u6c47", "(currency", "\u82f1\u9551", " Euros"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 4974.0, "target_rank": 1456.5, "source_logit": 2.8125, "target_logit": 5.375, "top_ids": [104142, 116866, 103408, 106333, 101237, 110065, 112397, 113883, 106127, 109442, 99599, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u6b27\u5143", "\u5916\u6c47", "\u7f8e\u5143", "\u6c47\u7387", "\u56fd\u5bb6\u6807\u51c6", "\u672c\u56fd", "\u6b27\u76df", "\u8d22\u653f\u90e8", "\u56fd\u5bb6", "\u6210\u5458\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 46.0, "target_rank": 5418.5, "source_logit": 14.625, "target_logit": 2.140625, "top_ids": [11413, 11192, 28453, 23734, 2598, 104142, 32488, 17692, 103408, 80189, 85466, 19551], "top_tokens": [" currency", " dollars", " Currency", " Turkish", " called", "\u4eba\u6c11\u5e01", " euros", " dollar", "\u6b27\u5143", " Dollars", " Euros", " USD"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 8.5, "target_rank": 8172.0, "source_logit": 18.875, "target_logit": 0.57421875, "top_ids": [23734, 11413, 104142, 28453, 17692, 17860, 11192, 17009, 17672, 109236, 19551, 35602], "top_tokens": [" Turkish", " currency", "\u4eba\u6c11\u5e01", " Currency", " dollar", " Greek", " dollars", " Turkey", " euro", "\u571f\u8033\u5176", " USD", " currencies"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 49.0, "target_rank": 2157.5, "source_logit": 22.875, "target_logit": 8.5, "top_ids": [11413, 104142, 28453, 11192, 103408, 35602, 32488, 80189, 112455, 17692, 26321, 34710], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " Currency", " dollars", "\u6b27\u5143", " currencies", " euros", " Dollars", "\u82f1\u9551", " dollar", "Currency", "_currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 24.0, "target_rank": 2526.0, "source_logit": 22.625, "target_logit": 5.40625, "top_ids": [104142, 23734, 11413, 35602, 103408, 28453, 110065, 101406, 101237, 26321, 17692, 34710], "top_tokens": ["\u4eba\u6c11\u5e01", " Turkish", " currency", " currencies", "\u6b27\u5143", " Currency", "\u6c47\u7387", "\u8d27\u5e01", "\u7f8e\u5143", "Currency", " dollar", "_currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 46.5, "target_rank": 7616.0, "source_logit": 16.5, "target_logit": 5.59375, "top_ids": [279, 23734, 2598, 19461, 7855, 3881, 326, 444, 3371, 220, 2987, 1083], "top_tokens": [" the", " Turkish", " called", " Euro", " British", " known", " l", " L", " den", " ", " either", " also"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 16.5, "target_rank": 4506.0, "source_logit": 17.125, "target_logit": 4.75, "top_ids": [23734, 326, 444, 19461, 17672, 7855, 17860, 109236, 8705, 27022, 69010, 809], "top_tokens": [" Turkish", " l", " L", " Euro", " euro", " British", " Greek", "\u571f\u8033\u5176", " Tur", " Lev", " Ottoman", " Y"]}], "source_token": 17009, "target_token": 29467, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Iz", "mir", " is", " the"], "seconds": 1.1596816251985729}
{"run_id": "eval-13/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.32822850346565247, "target_logp": -18.466083526611328, "log_odds": -18.137855023145676, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.0727665051817894, -0.2524951100349426, -0.002966885920614004]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-16.94776725769043, -1.5182907581329346, -2.5748875486897305e-05]}}, "score_trace": {"source": [], "target": []}, "output": " Turkish lira.  What is the currency used in the", "output_ids": [23734, 326, 8832, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [], "first_top_ids": [23734, 326, 444, 19461, 17672, 7855, 17860, 109236, 69010, 27022], "first_top_logits": [27.25, 24.125, 23.5, 22.5, 19.875, 19.75, 19.375, 19.0, 18.875, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.4925992921926081, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.32822850346565247, "target_logp": -18.466083526611328, "log_odds": -18.137855023145676, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.0727665051817894, -0.2524951100349426, -0.002966885920614004]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-16.94776725769043, -1.5182907581329346, -2.5748875486897305e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  What is the currency used in the", "output_ids": [23734, 326, 8832, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 326, 444, 19461, 17672, 7855, 17860, 109236, 69010, 27022], "first_top_logits": [27.25, 24.125, 23.5, 22.5, 19.875, 19.75, 19.375, 19.0, 18.875, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.4631593329831958, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -35.14859390258789, "target_logp": -7.382107257843018, "log_odds": 27.766486644744873, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-34.89244842529297, -0.25302785634994507, -0.0031163957901299]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-6.376824855804443, -1.0052690505981445, -1.3351351299206726e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.6155395507812, 807.0729370117188]], "actual_norm": [[377.03863525390625, 128.65000915527344, 61.7578125, 63.67232131958008, 159.98068237304688, 233.39610290527344, 205.8048858642578, 139.6795654296875, 36.54475784301758, 66.46759796142578, 678.2604370117188, 655.63525390625, 807.0261840820312]], "relative_norm": [[0.023839207366108894, 0.16195832192897797, 0.05343345180153847, 0.05608682334423065, 0.1506311446428299, 0.2158830463886261, 0.19820360839366913, 0.14080868661403656, 0.03810044378042221, 0.06071074679493904, 0.7379154562950134, 0.6099931597709656, 0.803358256816864]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.6155395507812, 807.0729370117188]], "actual_norm": [[377.03863525390625, 128.65000915527344, 61.7578125, 63.67232131958008, 159.98068237304688, 233.39610290527344, 205.8048858642578, 139.6795654296875, 36.54475784301758, 66.46759796142578, 678.2604370117188, 655.63525390625, 807.0261840820312]], "relative_norm": [[0.023839207366108894, 0.16195832192897797, 0.05343345180153847, 0.05608682334423065, 0.1506311446428299, 0.2158830463886261, 0.19820360839366913, 0.14080868661403656, 0.03810044378042221, 0.06071074679493904, 0.7379154562950134, 0.6099931597709656, 0.803358256816864]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thailand Baht.  What is the name of the country", "output_ids": [29467, 14322, 426, 13, 220, 3555, 374, 279, 829, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 135.03700256347656, 62.99478530883789, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 64.00763702392578, 680.3002319335938, 651.0289306640625, 798.3638916015625]], "actual_norm": [[377.03863525390625, 134.98385620117188, 63.00212478637695, 64.04742431640625, 157.68019104003906, 233.38232421875, 202.5654754638672, 138.906005859375, 32.78553771972656, 64.0589370727539, 680.3851318359375, 651.0406494140625, 798.3773803710938]], "relative_norm": [[0.023839207366108894, 0.17246253788471222, 0.054534394294023514, 0.056584682315588, 0.14827857911586761, 0.21626976132392883, 0.19556525349617004, 0.13960370421409607, 0.03411104530096054, 0.05864593759179115, 0.7405689358711243, 0.6081643104553223, 0.7996907234191895]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29467, 26392, 20539, 65431, 59016, 98867, 33796, 13369, 105906, 19180], "first_top_logits": [31.625, 25.25, 22.5, 20.625, 19.875, 19.375, 19.25, 19.125, 18.125, 17.875], "source_correct": false, "target_correct": true, "seconds": 0.46760741621255875, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.375981330871582, "target_logp": -18.368959426879883, "log_odds": -15.9929780960083, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-2.0237903594970703, -0.3488149046897888, -0.0033759051002562046]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-17.21129035949707, -1.1576495170593262, -1.8954096958623268e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.253173828125, 655.615478515625, 807.0729370117188]], "actual_norm": [[376.9759521484375, 128.65379333496094, 61.773521423339844, 63.724510192871094, 160.0164031982422, 233.35623168945312, 205.78457641601562, 139.67947387695312, 36.59098434448242, 66.37120056152344, 678.2576904296875, 655.643798828125, 807.1011962890625]], "relative_norm": [[0.02383524551987648, 0.16196309030056, 0.053447045385837555, 0.05613279715180397, 0.1506647765636444, 0.21584616601467133, 0.1981840580701828, 0.1408085972070694, 0.03814863786101341, 0.060622699558734894, 0.7379124164581299, 0.6100010871887207, 0.8034329414367676]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.253173828125, 655.615478515625, 807.0729370117188]], "actual_norm": [[376.9759521484375, 128.65379333496094, 61.773521423339844, 63.724510192871094, 160.0164031982422, 233.35623168945312, 205.78457641601562, 139.67947387695312, 36.59098434448242, 66.37120056152344, 678.2576904296875, 655.643798828125, 807.1011962890625]], "relative_norm": [[0.02383524551987648, 0.16196309030056, 0.053447045385837555, 0.05613279715180397, 0.1506647765636444, 0.21584616601467133, 0.1981840580701828, 0.1408085972070694, 0.03814863786101341, 0.060622699558734894, 0.7379124164581299, 0.6100010871887207, 0.8034329414367676]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Iz", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 47823], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 135.03700256347656, 62.994789123535156, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 64.00763702392578, 680.3002319335938, 651.0289306640625, 798.3638916015625]], "actual_norm": [[376.9759521484375, 135.00796508789062, 63.02482223510742, 64.10845947265625, 157.65841674804688, 233.33932495117188, 202.6165008544922, 138.87081909179688, 32.8104362487793, 63.99604034423828, 680.3787841796875, 651.11328125, 798.34765625]], "relative_norm": [[0.02383524551987648, 0.1724933534860611, 0.0545540414750576, 0.056638605892658234, 0.14825810492038727, 0.21622991561889648, 0.1956145167350769, 0.13956834375858307, 0.034136950969696045, 0.05858835205435753, 0.7405620217323303, 0.6082322001457214, 0.7996609807014465]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 23734, 6633, 29615, 69010, 109236, 220, 17860, 19551, 19461], "first_top_logits": [27.875, 26.0, 21.875, 20.25, 20.125, 20.0, 19.875, 19.5, 19.375, 19.375], "source_correct": false, "target_correct": false, "seconds": 0.4486228749155998, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.8313751220703125, "target_logp": -43.79395294189453, "log_odds": -36.96257781982422, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-6.574404716491699, -0.25229740142822266, -0.004673035815358162]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-42.074405670166016, -1.7194979190826416, -4.9232225137529895e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.6155395507812, 807.0729370117188]], "actual_norm": [[377.0504455566406, 128.62863159179688, 61.74644088745117, 63.705318450927734, 159.96087646484375, 233.3307342529297, 205.8165283203125, 139.7642364501953, 36.532752990722656, 66.48294830322266, 678.2766723632812, 655.6559448242188, 807.1002807617188]], "relative_norm": [[0.023839954286813736, 0.16193141043186188, 0.05342361330986023, 0.05611589178442955, 0.1506124883890152, 0.21582257747650146, 0.1982148289680481, 0.14089404046535492, 0.03808792680501938, 0.06072476878762245, 0.7379330992698669, 0.610012412071228, 0.803432047367096]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453186035156, 63.68741226196289, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.6155395507812, 807.0729370117188]], "actual_norm": [[377.0504455566406, 128.62863159179688, 61.74644088745117, 63.705318450927734, 159.96087646484375, 233.3307342529297, 205.8165283203125, 139.7642364501953, 36.532752990722656, 66.48294830322266, 678.2766723632812, 655.6559448242188, 807.1002807617188]], "relative_norm": [[0.023839954286813736, 0.16193141043186188, 0.05342361330986023, 0.05611589178442955, 0.1506124883890152, 0.21582257747650146, 0.1982148289680481, 0.14089404046535492, 0.03808792680501938, 0.06072476878762245, 0.7379330992698669, 0.610012412071228, 0.803432047367096]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkey lira.  What would be a good title for", "output_ids": [17009, 326, 8832, 13, 220, 3555, 1035, 387, 264, 1661, 2265, 369], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 135.03700256347656, 62.99478530883789, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 64.00763702392578, 680.3002319335938, 651.0289306640625, 798.3638916015625]], "actual_norm": [[377.0504455566406, 135.05885314941406, 63.02458190917969, 64.05262756347656, 157.59274291992188, 233.28236389160156, 202.58493041992188, 138.8513641357422, 32.75849533081055, 64.03720092773438, 680.3382568359375, 651.1358642578125, 798.325927734375]], "relative_norm": [[0.023839954286813736, 0.17255836725234985, 0.05455383285880089, 0.0565892793238163, 0.14819635450839996, 0.21617713570594788, 0.19558404386043549, 0.13954877853393555, 0.034082911908626556, 0.05862603709101677, 0.7405179142951965, 0.6082532405853271, 0.7996392250061035]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17009, 444, 326, 23734, 44305, 350, 8705, 2613, 19458, 330], "first_top_logits": [26.375, 23.625, 21.625, 19.875, 18.5, 17.5, 17.25, 17.125, 17.125, 17.125], "source_correct": true, "target_correct": false, "seconds": 0.4392329170368612, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.42761924862861633, "target_logp": -15.385566711425781, "log_odds": -14.957947462797165, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.1096731424331665, -0.3141997158527374, -0.003746397327631712]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-14.047173500061035, -1.3383713960647583, -2.1576648578047752e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 128.63661193847656, 61.7545280456543, 63.687408447265625, 159.97434997558594, 233.39041137695312, 205.8092041015625, 139.7261199951172, 36.54576873779297, 66.38880157470703, 678.2531127929688, 655.615478515625, 807.0729370117188]], "actual_norm": [[376.4582214355469, 128.68609619140625, 61.753543853759766, 63.66917037963867, 159.94149780273438, 233.38253784179688, 205.7348175048828, 139.74765014648438, 36.566001892089844, 66.46031188964844, 678.3347778320312, 655.5383911132812, 807.0464477539062]], "relative_norm": [[0.023802509531378746, 0.162003755569458, 0.05342976003885269, 0.0560840480029583, 0.15059424936771393, 0.21587048470973969, 0.1981361359357834, 0.14087732136249542, 0.03812259063124657, 0.06070409342646599, 0.7379963397979736, 0.6099030375480652, 0.8033784627914429]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 128.63661193847656, 61.7545280456543, 63.687408447265625, 159.97434997558594, 233.39041137695312, 205.8092041015625, 139.7261199951172, 36.54576873779297, 66.38880157470703, 678.2531127929688, 655.615478515625, 807.0729370117188]], "actual_norm": [[376.4582214355469, 128.68609619140625, 61.753543853759766, 63.66917037963867, 159.94149780273438, 233.38253784179688, 205.7348175048828, 139.74765014648438, 36.566001892089844, 66.46031188964844, 678.3347778320312, 655.5383911132812, 807.0464477539062]], "relative_norm": [[0.023802509531378746, 0.162003755569458, 0.05342976003885269, 0.0560840480029583, 0.15059424936771393, 0.21587048470973969, 0.1981361359357834, 0.14087732136249542, 0.03812259063124657, 0.06070409342646599, 0.7379963397979736, 0.6099030375480652, 0.8033784627914429]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  What is the currency used in the", "output_ids": [23734, 326, 8832, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 135.0369873046875, 62.994781494140625, 64.02460479736328, 157.65682983398438, 233.3636016845703, 202.6132049560547, 138.8590087890625, 32.77949142456055, 64.00762939453125, 680.3002319335938, 651.0288696289062, 798.3638916015625]], "actual_norm": [[376.4582214355469, 135.1025390625, 62.98004913330078, 63.975440979003906, 157.5896759033203, 233.31263732910156, 202.5191192626953, 138.84823608398438, 32.79201126098633, 64.00225067138672, 680.3174438476562, 651.0415649414062, 798.4210205078125]], "relative_norm": [[0.023802509531378746, 0.1726141721010208, 0.0545152872800827, 0.056521087884902954, 0.14819346368312836, 0.2162051945924759, 0.19552050530910492, 0.13954563438892365, 0.03411778062582016, 0.05859404057264328, 0.7404952645301819, 0.6081652045249939, 0.7997344732284546]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 19461, 17672, 7855, 17860, 8705, 69010, 109236, 3639, 20545], "first_top_logits": [25.375, 23.0, 21.0, 20.125, 19.25, 19.0, 18.0, 17.875, 17.125, 16.75], "source_correct": true, "target_correct": false, "seconds": 0.4437959589995444, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1992714405059814, "target_logp": -18.29358673095703, "log_odds": -17.09431529045105, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.9682697057723999, -0.22751078009605408, -0.003490904113277793]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-16.87451934814453, -1.4190479516983032, -1.9192511899746023e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453567504883, 63.687416076660156, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880920410156, 678.253173828125, 655.6156005859375, 807.0729370117188]], "actual_norm": [[376.8459167480469, 128.59649658203125, 61.74306869506836, 63.68623352050781, 160.00210571289062, 233.35777282714844, 205.8319091796875, 139.75733947753906, 36.56819534301758, 66.46896362304688, 678.2938842773438, 655.6632690429688, 807.0662841796875]], "relative_norm": [[0.023827023804187775, 0.16189095377922058, 0.053420696407556534, 0.05609907954931259, 0.15065132081508636, 0.21584758162498474, 0.19822964072227478, 0.1408870965242386, 0.03812487795948982, 0.060711994767189026, 0.7379518151283264, 0.6100192070007324, 0.8033981919288635]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 128.63662719726562, 61.75453567504883, 63.687416076660156, 159.97434997558594, 233.3904266357422, 205.80921936035156, 139.7261199951172, 36.545772552490234, 66.38880920410156, 678.253173828125, 655.6156005859375, 807.0729370117188]], "actual_norm": [[376.8459167480469, 128.59649658203125, 61.74306869506836, 63.68623352050781, 160.00210571289062, 233.35777282714844, 205.8319091796875, 139.75733947753906, 36.56819534301758, 66.46896362304688, 678.2938842773438, 655.6632690429688, 807.0662841796875]], "relative_norm": [[0.023827023804187775, 0.16189095377922058, 0.053420696407556534, 0.05609907954931259, 0.15065132081508636, 0.21584758162498474, 0.19822964072227478, 0.1408870965242386, 0.03812487795948982, 0.060711994767189026, 0.7379518151283264, 0.6100192070007324, 0.8033981919288635]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Euro.  What is the currency used in the country containing", "output_ids": [19461, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.985107421875, 135.03700256347656, 62.994789123535156, 64.02461242675781, 157.65682983398438, 233.36363220214844, 202.61322021484375, 138.85902404785156, 32.77949523925781, 64.00763702392578, 680.3002319335938, 651.0289306640625, 798.3639526367188]], "actual_norm": [[376.8459167480469, 135.0237274169922, 63.00617599487305, 64.029541015625, 157.68714904785156, 233.34683227539062, 202.6534423828125, 138.89967346191406, 32.82101058959961, 64.0038070678711, 680.3050537109375, 650.98779296875, 798.3663940429688]], "relative_norm": [[0.023827023804187775, 0.17251348495483398, 0.054537903517484665, 0.05656888335943222, 0.1482851207256317, 0.216236874461174, 0.19565019011497498, 0.1395973414182663, 0.0341479554772377, 0.05859546363353729, 0.7404817342758179, 0.6081149578094482, 0.7996797561645508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [19461, 23734, 17672, 7513, 8705, 37088, 13, 468, 71757, 13232], "first_top_logits": [23.5, 23.125, 21.875, 19.25, 17.625, 17.375, 17.375, 17.125, 17.0, 17.0], "source_correct": false, "target_correct": false, "seconds": 0.4167124158702791, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-13/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-13", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Izmir", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Izmir is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.3526700735092163, "target_logp": -17.267711639404297, "log_odds": -16.91504156589508, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.12375276535749435, -0.22603993117809296, -0.0028773832600563765]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-15.93625259399414, -1.331427812576294, -3.111314072157256e-05]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 128.63662719726562, 61.7545280456543, 63.687408447265625, 159.97433471679688, 233.39041137695312, 205.8092041015625, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.615478515625, 807.0728759765625]], "actual_norm": [[376.5677185058594, 128.5969696044922, 61.70604705810547, 63.635780334472656, 159.98367309570312, 233.41004943847656, 205.85841369628906, 139.72706604003906, 36.626869201660156, 66.45259857177734, 678.217041015625, 655.499755859375, 807.0366821289062]], "relative_norm": [[0.023809432983398438, 0.16189154982566833, 0.05338866263628006, 0.05605463683605194, 0.15063396096229553, 0.21589593589305878, 0.19825516641139984, 0.1408565789461136, 0.038186050951480865, 0.0606970489025116, 0.7378681898117065, 0.6098670959472656, 0.8033686876296997]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 128.63662719726562, 61.7545280456543, 63.687408447265625, 159.97433471679688, 233.39041137695312, 205.8092041015625, 139.7261199951172, 36.545772552490234, 66.38880157470703, 678.2531127929688, 655.615478515625, 807.0728759765625]], "actual_norm": [[376.5677185058594, 128.5969696044922, 61.70604705810547, 63.635780334472656, 159.98367309570312, 233.41004943847656, 205.85841369628906, 139.72706604003906, 36.626869201660156, 66.45259857177734, 678.217041015625, 655.499755859375, 807.0366821289062]], "relative_norm": [[0.023809432983398438, 0.16189154982566833, 0.05338866263628006, 0.05605463683605194, 0.15063396096229553, 0.21589593589305878, 0.19825516641139984, 0.1408565789461136, 0.038186050951480865, 0.0606970489025116, 0.7378681898117065, 0.6098670959472656, 0.8033686876296997]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  What would be the best way to", "output_ids": [23734, 326, 8832, 13, 220, 3555, 1035, 387, 279, 1850, 1616, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 47823, 19936, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[376.9850769042969, 135.03700256347656, 62.99478530883789, 64.02460479736328, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 64.00763702392578, 680.3002319335938, 651.0288696289062, 798.3638916015625]], "actual_norm": [[376.5677185058594, 134.9921112060547, 62.94397735595703, 63.99387741088867, 157.6293182373047, 233.35350036621094, 202.66351318359375, 138.87538146972656, 32.84272384643555, 63.96235275268555, 680.3076782226562, 651.0720825195312, 798.3240966796875]], "relative_norm": [[0.023809432983398438, 0.17247308790683746, 0.054484061896800995, 0.05653737485408783, 0.14823074638843536, 0.21624305844306946, 0.19565990567207336, 0.13957291841506958, 0.034170545637607574, 0.05855751410126686, 0.7404845952987671, 0.6081936955451965, 0.7996373772621155]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 444, 17860, 809, 326, 19461, 17009, 3639, 7855, 2274], "first_top_logits": [25.0, 22.375, 20.375, 19.625, 19.0, 18.875, 18.625, 18.5, 18.375, 18.125], "source_correct": true, "target_correct": false, "seconds": 0.4665113748051226, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/readout", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 737.0, "target_rank": 932.0, "source_logit": 3.109375, "target_logit": 2.796875, "top_ids": [8330, 38352, 11413, 15973, 7513, 45418, 3693, 6364, 4505, 7748, 31064, 31491], "top_tokens": [" Australia", "uenta", " currency", "currency", " European", ".currency", " American", " English", " Europe", " Indian", " Argentina", "uesto"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 2409.0, "target_rank": 2273.5, "source_logit": 1.9140625, "target_logit": 1.9921875, "top_ids": [11413, 15973, 96694, 53661, 3693, 3001, 18174, 5088, 28130, 28453, 4787, 42189], "top_tokens": [" currency", "currency", "onis", "cla", " American", "ica", "oret", "urrency", "ija", " Currency", " North", "etz"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 222.5, "target_rank": 310.5, "source_logit": 17.375, "target_logit": 16.75, "top_ids": [11413, 35602, 7513, 7214, 6489, 32488, 90475, 24357, 36445, 56684, 85466, 8330], "top_tokens": [" currency", " currencies", " European", " foreign", " international", " euros", "(currency", " overseas", " internationally", " Cyprus", " Euros", " Australia"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 798.5, "target_rank": 798.5, "source_logit": 12.0, "target_logit": 12.0, "top_ids": [11413, 7214, 65001, 35602, 47412, 21130, 37198, 13224, 74508, 7855, 78253, 5313], "top_tokens": [" currency", " foreign", "\"\u2014", " currencies", " foreigners", " parliament", " Commonwealth", " Britain", "\u00bb.", " British", " treasury", " national"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 3590.5, "target_rank": 2809.0, "source_logit": 0.8359375, "target_logit": 1.171875, "top_ids": [21251, 28453, 3776, 3731, 603, 5737, 55964, 12751, 7513, 11413, 19833, 21525], "top_tokens": ["onia", " Currency", " One", " Red", "one", " Group", "\u2011", " Roman", " European", " currency", " Amb", " Ring"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1054.5, "target_rank": 6922.0, "source_logit": 2.28125, "target_logit": -0.34375, "top_ids": [7179, 11413, 18174, 11766, 364, 21251, 29991, 479, 3639, 64, 2932, 105469], "top_tokens": [" International", " currency", "oret", " Standard", " '", "onia", "\u540d\u79f0", " G", " United", "a", " She", "\u79cd\u7c7b"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 843.0, "target_rank": 1706.5, "source_logit": 13.3125, "target_logit": 12.3125, "top_ids": [11413, 35602, 3146, 5313, 5837, 7576, 13867, 43895, 28453, 3033, 3283, 7513], "top_tokens": [" currency", " currencies", " country", " national", " countries", " prices", " county", " tariffs", " Currency", " government", " city", " European"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 507.0, "target_rank": 428.5, "source_logit": 12.6875, "target_logit": 12.9375, "top_ids": [11413, 37198, 7513, 5313, 7855, 6489, 5837, 7214, 105492, 94936, 35602, 3146], "top_tokens": [" currency", " Commonwealth", " European", " national", " British", " international", " countries", " foreign", "\u4e2d\u534e\u4eba\u6c11", " denomination", " currencies", " country"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3830.5, "target_rank": 6591.0, "source_logit": 0.44921875, "target_logit": -0.4140625, "top_ids": [3693, 2240, 6364, 4787, 89478, 279, 1083, 2598, 271, 4102, 5434, 15503], "top_tokens": [" American", " None", " English", " North", "\u2026\n", " the", " also", " called", "\n\n", "\u00a0", "\u2026\n\n", " primarily"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 1996.5, "target_rank": 8328.0, "source_logit": 1.515625, "target_logit": -0.7890625, "top_ids": [1852, 3693, 5938, 3639, 4787, 479, 271, 7179, 4102, 4882, 5055, 730], "top_tokens": [" same", " American", " German", " United", " North", " G", "\n\n", " International", "\u00a0", " South", " National", " K"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1219.0, "target_rank": 1404.0, "source_logit": 12.1875, "target_logit": 11.875, "top_ids": [11413, 35602, 104142, 103408, 113883, 28453, 106333, 32488, 115160, 113596, 26321, 90475], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u672c\u56fd", " Currency", "\u5916\u6c47", " euros", "\u5176\u4ed6\u56fd\u5bb6", "\u56fd\u7c4d", "Currency", "(currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 3137.0, "target_rank": 2247.5, "source_logit": 4.78125, "target_logit": 5.5, "top_ids": [104142, 116866, 113883, 116588, 112397, 103408, 106127, 106333, 110065, 109442, 104773, 101237], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u672c\u56fd", "\u6210\u5458\u56fd", "\u56fd\u5bb6\u6807\u51c6", "\u6b27\u5143", "\u6b27\u76df", "\u5916\u6c47", "\u6c47\u7387", "\u8d22\u653f\u90e8", "\u5171\u548c\u56fd", "\u7f8e\u5143"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 32.0, "target_rank": 8396.0, "source_logit": 15.6875, "target_logit": 1.1484375, "top_ids": [11413, 11192, 23734, 17692, 104142, 28453, 80189, 32488, 77186, 35602, 19551, 85466], "top_tokens": [" currency", " dollars", " Turkish", " dollar", "\u4eba\u6c11\u5e01", " Currency", " Dollars", " euros", ".Currency", " currencies", " USD", " Euros"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 17573.0, "source_logit": 21.25, "target_logit": -0.55078125, "top_ids": [23734, 11413, 109236, 17009, 17692, 104142, 28453, 11192, 17672, 69010, 80189, 35602], "top_tokens": [" Turkish", " currency", "\u571f\u8033\u5176", " Turkey", " dollar", "\u4eba\u6c11\u5e01", " Currency", " dollars", " euro", " Ottoman", " Dollars", " currencies"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 37.0, "target_rank": 2741.5, "source_logit": 26.125, "target_logit": 8.8125, "top_ids": [11413, 11192, 104142, 35602, 28453, 80189, 17692, 32488, 103408, 77186, 101406, 101237], "top_tokens": [" currency", " dollars", "\u4eba\u6c11\u5e01", " currencies", " Currency", " Dollars", " dollar", " euros", "\u6b27\u5143", ".Currency", "\u8d27\u5e01", "\u7f8e\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 8.0, "target_rank": 4995.5, "source_logit": 24.375, "target_logit": 3.5625, "top_ids": [23734, 104142, 11413, 109236, 35602, 28453, 70244, 17009, 101406, 110065, 77186, 34710], "top_tokens": [" Turkish", "\u4eba\u6c11\u5e01", " currency", "\u571f\u8033\u5176", " currencies", " Currency", " Ankara", " Turkey", "\u8d27\u5e01", "\u6c47\u7387", ".Currency", "_currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 34.0, "target_rank": 12176.0, "source_logit": 17.25, "target_logit": 4.375, "top_ids": [279, 23734, 326, 2598, 7855, 19461, 3639, 444, 2987, 2274, 220, 109236], "top_tokens": [" the", " Turkish", " l", " called", " British", " Euro", " United", " L", " either", " US", " ", "\u571f\u8033\u5176"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 9.0, "target_rank": 10940.0, "source_logit": 18.875, "target_logit": 2.421875, "top_ids": [23734, 326, 109236, 444, 69010, 20545, 8705, 3639, 17009, 95564, 19461, 27022], "top_tokens": [" Turkish", " l", "\u571f\u8033\u5176", " L", " Ottoman", " Turk", " Tur", " United", " Turkey", " Anat", " Euro", " Lev"]}], "source_token": 17009, "target_token": 29467, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Ankara", " is", " the"], "seconds": 1.154269332997501}
{"run_id": "eval-14/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.2935124933719635, "target_logp": -20.713726043701172, "log_odds": -20.42021355032921, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.007645038887858391, -0.28240352869033813, -0.0034639376681298018]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-19.945144653320312, -0.768561065196991, -2.074220174108632e-05]}}, "score_trace": {"source": [], "target": []}, "output": " Turkish lira.  Inference: The country is Turkey", "output_ids": [23734, 326, 8832, 13, 220, 758, 2202, 25, 576, 3146, 374, 17009], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [], "first_top_ids": [23734, 326, 109236, 444, 69010, 20545, 8705, 3639, 17009, 95564], "first_top_logits": [27.5, 22.25, 20.25, 20.0, 19.875, 19.25, 19.125, 19.0, 18.875, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.4774026251398027, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.2935124933719635, "target_logp": -20.713726043701172, "log_odds": -20.42021355032921, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.007645038887858391, -0.28240352869033813, -0.0034639376681298018]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-19.945144653320312, -0.768561065196991, -2.074220174108632e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  Inference: The country is Turkey", "output_ids": [23734, 326, 8832, 13, 220, 758, 2202, 25, 576, 3146, 374, 17009], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 326, 109236, 444, 69010, 20545, 8705, 3639, 17009, 95564], "first_top_logits": [27.5, 22.25, 20.25, 20.0, 19.875, 19.25, 19.125, 19.0, 18.875, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.48233200004324317, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -37.006248474121094, "target_logp": -8.126405715942383, "log_odds": 28.87984275817871, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-36.71930694580078, -0.2842364013195038, -0.0027053682133555412]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-7.625556945800781, -0.5008348226547241, -1.4662635294371285e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[377.03863525390625, 128.30166625976562, 61.77170181274414, 64.94261169433594, 158.4276580810547, 231.0460968017578, 205.59068298339844, 138.6269989013672, 33.89531326293945, 727.3828125, 790.3799438476562, 1053.23046875]], "relative_norm": [[0.023839207366108894, 0.16192558407783508, 0.05332551151514053, 0.057274091988801956, 0.1492760181427002, 0.21372543275356293, 0.19816641509532928, 0.13967998325824738, 0.035246510058641434, 0.7573204636573792, 0.7120834589004517, 0.9785496592521667]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[377.03863525390625, 128.30166625976562, 61.77170181274414, 64.94261169433594, 158.4276580810547, 231.0460968017578, 205.59068298339844, 138.6269989013672, 33.89531326293945, 727.3828125, 790.3799438476562, 1053.23046875]], "relative_norm": [[0.023839207366108894, 0.16192558407783508, 0.05332551151514053, 0.057274091988801956, 0.1492760181427002, 0.21372543275356293, 0.19816641509532928, 0.13967998325824738, 0.035246510058641434, 0.7573204636573792, 0.7120834589004517, 0.9785496592521667]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thailand Thaattan.  What is the name of", "output_ids": [29467, 663, 64, 1587, 276, 13, 220, 3555, 374, 279, 829, 315], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 135.03700256347656, 62.99478530883789, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 730.5884399414062, 790.3417358398438, 1046.9466552734375]], "actual_norm": [[377.03863525390625, 134.98385620117188, 63.00212478637695, 64.04742431640625, 157.68019104003906, 233.38232421875, 202.5654754638672, 138.906005859375, 32.78553771972656, 730.5359497070312, 790.3504028320312, 1046.8603515625]], "relative_norm": [[0.023839207366108894, 0.17246253788471222, 0.054534394294023514, 0.056584682315588, 0.14827857911586761, 0.21626976132392883, 0.19556525349617004, 0.13960370421409607, 0.03411104530096054, 0.7614147663116455, 0.7078529000282288, 0.9767263531684875]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29467, 26392, 59016, 105906, 20539, 65431, 98867, 3240, 61038, 58829], "first_top_logits": [30.125, 22.625, 19.875, 18.5, 18.375, 17.75, 17.25, 17.0, 16.875, 16.5], "source_correct": false, "target_correct": false, "seconds": 0.46717200009152293, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.324739933013916, "target_logp": -19.74932861328125, "log_odds": -16.424588680267334, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-2.934126377105713, -0.38763490319252014, -0.00297853397205472]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-19.309125900268555, -0.4401865005493164, -1.5139465176616795e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.234130859375]], "actual_norm": [[376.9759521484375, 128.34288024902344, 61.80657196044922, 64.95709228515625, 158.38087463378906, 231.09774780273438, 205.61358642578125, 138.63182067871094, 33.87193298339844, 727.4085083007812, 790.4257202148438, 1053.3140869140625]], "relative_norm": [[0.02383524551987648, 0.1619776040315628, 0.053355615586042404, 0.057286862283945084, 0.1492319256067276, 0.2137732207775116, 0.19818848371505737, 0.13968484103679657, 0.03522219508886337, 0.7573472261428833, 0.7121247053146362, 0.9786273241043091]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.234130859375]], "actual_norm": [[376.9759521484375, 128.34288024902344, 61.80657196044922, 64.95709228515625, 158.38087463378906, 231.09774780273438, 205.61358642578125, 138.63182067871094, 33.87193298339844, 727.4085083007812, 790.4257202148438, 1053.3140869140625]], "relative_norm": [[0.02383524551987648, 0.1619776040315628, 0.053355615586042404, 0.057286862283945084, 0.1492319256067276, 0.2137732207775116, 0.19818848371505737, 0.13968484103679657, 0.03522219508886337, 0.7573472261428833, 0.7121247053146362, 0.9786273241043091]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  In the country containing Ankara, the currency is", "output_ids": [1965, 13, 220, 758, 279, 3146, 8482, 70244, 11, 279, 11413, 374], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 135.03700256347656, 62.994789123535156, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 730.5884399414062, 790.341796875, 1046.9466552734375]], "actual_norm": [[376.9759521484375, 135.00796508789062, 63.02482223510742, 64.10845947265625, 157.65841674804688, 233.33932495117188, 202.6165008544922, 138.87081909179688, 32.8104362487793, 730.6328735351562, 790.2701416015625, 1046.8560791015625]], "relative_norm": [[0.02383524551987648, 0.1724933534860611, 0.0545540414750576, 0.056638605892658234, 0.14825810492038727, 0.21622991561889648, 0.1956145167350769, 0.13956834375858307, 0.034136950969696045, 0.7615157961845398, 0.7077810168266296, 0.9767223596572876]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 23734, 6633, 69010, 1140, 3146, 220, 659, 109236, 17009], "first_top_logits": [27.75, 24.875, 21.125, 20.125, 19.625, 19.375, 19.25, 19.0, 18.875, 18.875], "source_correct": false, "target_correct": false, "seconds": 0.4497454580850899, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -9.77784538269043, "target_logp": -47.4659538269043, "log_odds": -37.68810844421387, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-9.385899543762207, -0.3875862956047058, -0.004359384998679161]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-46.76089859008789, -0.7050261497497559, -3.0040289857424796e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[377.0504455566406, 128.2926788330078, 61.768638610839844, 64.97396087646484, 158.3549041748047, 231.05645751953125, 205.54849243164062, 138.5816192626953, 33.92555236816406, 727.4776611328125, 790.513427734375, 1053.4285888671875]], "relative_norm": [[0.023839954286813736, 0.16191424429416656, 0.053322866559028625, 0.05730174109339714, 0.1492074579000473, 0.21373502910137177, 0.19812574982643127, 0.13963426649570465, 0.03527795150876045, 0.757419228553772, 0.7122036814689636, 0.9787337183952332]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.765445709228516, 64.9382095336914, 158.38278198242188, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[377.0504455566406, 128.2926788330078, 61.768638610839844, 64.97396087646484, 158.3549041748047, 231.05645751953125, 205.54849243164062, 138.5816192626953, 33.92555236816406, 727.4776611328125, 790.513427734375, 1053.4285888671875]], "relative_norm": [[0.023839954286813736, 0.16191424429416656, 0.053322866559028625, 0.05730174109339714, 0.1492074579000473, 0.21373502910137177, 0.19812574982643127, 0.13963426649570465, 0.03527795150876045, 0.757419228553772, 0.7122036814689636, 0.9787337183952332]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkey lira.  Inference: The country is Turkey", "output_ids": [17009, 326, 8832, 13, 220, 758, 2202, 25, 576, 3146, 374, 17009], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 135.03700256347656, 62.99478530883789, 64.02461242675781, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 730.5884399414062, 790.3417358398438, 1046.9466552734375]], "actual_norm": [[377.0504455566406, 135.05885314941406, 63.02458190917969, 64.05262756347656, 157.59274291992188, 233.28236389160156, 202.58493041992188, 138.8513641357422, 32.75849533081055, 730.6473388671875, 790.4017333984375, 1046.8956298828125]], "relative_norm": [[0.023839954286813736, 0.17255836725234985, 0.05455383285880089, 0.0565892793238163, 0.14819635450839996, 0.21617713570594788, 0.19558404386043549, 0.13954877853393555, 0.034082911908626556, 0.7615308165550232, 0.7078988552093506, 0.9767592549324036]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17009, 444, 326, 44305, 23734, 19458, 362, 350, 72719, 2411], "first_top_logits": [27.375, 22.625, 21.5, 19.375, 18.0, 17.75, 17.375, 17.125, 17.0, 17.0], "source_correct": true, "target_correct": false, "seconds": 0.45662670908495784, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.37753236293792725, "target_logp": -13.860054016113281, "log_odds": -13.482521653175354, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.02400442771613598, -0.34995198249816895, -0.0035759571474045515]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-13.149003982543945, -0.7110278010368347, -2.1934269170742482e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 128.32003784179688, 61.76544189453125, 64.9382095336914, 158.3827667236328, 231.08956909179688, 205.5840606689453, 138.60227966308594, 33.88273620605469, 727.382568359375, 790.4065551757812, 1053.234130859375]], "actual_norm": [[376.4582214355469, 128.34593200683594, 61.75859832763672, 64.9197006225586, 158.4593963623047, 231.07728576660156, 205.51284790039062, 138.62339782714844, 33.993404388427734, 727.2922973632812, 790.3547973632812, 1053.205078125]], "relative_norm": [[0.023802509531378746, 0.16198144853115082, 0.0533142015337944, 0.0572538860142231, 0.14930590987205505, 0.2137542963027954, 0.19809138774871826, 0.13967636227607727, 0.03534850850701332, 0.7572262287139893, 0.712060809135437, 0.9785260558128357]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 128.32003784179688, 61.76544189453125, 64.9382095336914, 158.3827667236328, 231.08956909179688, 205.5840606689453, 138.60227966308594, 33.88273620605469, 727.382568359375, 790.4065551757812, 1053.234130859375]], "actual_norm": [[376.4582214355469, 128.34593200683594, 61.75859832763672, 64.9197006225586, 158.4593963623047, 231.07728576660156, 205.51284790039062, 138.62339782714844, 33.993404388427734, 727.2922973632812, 790.3547973632812, 1053.205078125]], "relative_norm": [[0.023802509531378746, 0.16198144853115082, 0.0533142015337944, 0.0572538860142231, 0.14930590987205505, 0.2137542963027954, 0.19809138774871826, 0.13967636227607727, 0.03534850850701332, 0.7572262287139893, 0.712060809135437, 0.9785260558128357]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  What is the currency of the country", "output_ids": [23734, 326, 8832, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 135.0369873046875, 62.994781494140625, 64.02460479736328, 157.65682983398438, 233.3636016845703, 202.6132049560547, 138.8590087890625, 32.77949142456055, 730.5884399414062, 790.3416748046875, 1046.9466552734375]], "actual_norm": [[376.4582214355469, 135.1025390625, 62.98004913330078, 63.975440979003906, 157.5896759033203, 233.31263732910156, 202.5191192626953, 138.84823608398438, 32.79201126098633, 730.648681640625, 790.2471313476562, 1046.9168701171875]], "relative_norm": [[0.023802509531378746, 0.1726141721010208, 0.0545152872800827, 0.056521087884902954, 0.14819346368312836, 0.2162051945924759, 0.19552050530910492, 0.13954563438892365, 0.03411778062582016, 0.7615322470664978, 0.7077603936195374, 0.9767791032791138]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 19461, 8705, 17672, 109236, 20545, 69010, 7855, 17860, 8585], "first_top_logits": [24.75, 20.125, 19.125, 18.625, 18.25, 17.625, 17.375, 17.25, 17.125, 17.0], "source_correct": true, "target_correct": false, "seconds": 0.4238494159653783, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5199699997901917, "target_logp": -17.285551071166992, "log_odds": -16.7655810713768, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.26008594036102295, -0.2564237415790558, -0.003460373729467392]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-16.697586059570312, -0.5879483819007874, -1.7881233361549675e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.76544952392578, 64.93821716308594, 158.38279724121094, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[376.8459167480469, 128.29722595214844, 61.753273010253906, 64.96979522705078, 158.42140197753906, 231.06613159179688, 205.59133911132812, 138.66705322265625, 33.876502990722656, 727.3110961914062, 790.2949829101562, 1053.18017578125]], "relative_norm": [[0.023827023804187775, 0.1619199812412262, 0.0533096045255661, 0.05729806423187256, 0.14927011728286743, 0.21374396979808807, 0.19816704094409943, 0.1397203505039215, 0.03522694855928421, 0.7572457790374756, 0.7120069265365601, 0.9785029292106628]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 128.32005310058594, 61.76544952392578, 64.93821716308594, 158.38279724121094, 231.08958435058594, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4066162109375, 1053.2342529296875]], "actual_norm": [[376.8459167480469, 128.29722595214844, 61.753273010253906, 64.96979522705078, 158.42140197753906, 231.06613159179688, 205.59133911132812, 138.66705322265625, 33.876502990722656, 727.3110961914062, 790.2949829101562, 1053.18017578125]], "relative_norm": [[0.023827023804187775, 0.1619199812412262, 0.0533096045255661, 0.05729806423187256, 0.14927011728286743, 0.21374396979808807, 0.19816704094409943, 0.1397203505039215, 0.03522694855928421, 0.7572457790374756, 0.7120069265365601, 0.9785029292106628]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  Inference: The country is Turkey", "output_ids": [23734, 326, 8832, 13, 220, 758, 2202, 25, 576, 3146, 374, 17009], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.985107421875, 135.03700256347656, 62.994789123535156, 64.02461242675781, 157.65682983398438, 233.36363220214844, 202.61322021484375, 138.85902404785156, 32.77949523925781, 730.5884399414062, 790.341796875, 1046.9466552734375]], "actual_norm": [[376.8459167480469, 135.0237274169922, 63.00617599487305, 64.029541015625, 157.68714904785156, 233.34683227539062, 202.6534423828125, 138.89967346191406, 32.82101058959961, 730.6790771484375, 790.3733520507812, 1046.9801025390625]], "relative_norm": [[0.023827023804187775, 0.17251348495483398, 0.054537903517484665, 0.05656888335943222, 0.1482851207256317, 0.216236874461174, 0.19565019011497498, 0.1395973414182663, 0.0341479554772377, 0.7615638971328735, 0.7078734636306763, 0.9768380522727966]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 19461, 17672, 13, 7513, 17009, 69010, 11413, 1532, 8705], "first_top_logits": [22.125, 20.0, 19.75, 18.25, 17.5, 17.25, 16.5, 16.5, 16.5, 16.25], "source_correct": true, "target_correct": false, "seconds": 0.4839722909964621, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-14/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-14", "split": "eval", "country": "Turkey", "currency": "Turkish lira", "aliases": ["lira", "TRY"], "city": "Ankara", "target_country": "Thailand", "target_currency": "Thai baht", "target_aliases": ["baht", "THB"], "prompt": "Fact: The currency used in the country containing Ankara is the", "source_continuation": " Turkish lira", "target_continuation": " Thai baht"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.37212541699409485, "target_logp": -18.3284912109375, "log_odds": -17.956365793943405, "token_scores": {"source": {"ids": [23734, 326, 8832], "log_probs": [-0.08704625815153122, -0.28241443634033203, -0.0026647080667316914]}, "target": {"ids": [26392, 12789, 426], "log_probs": [-17.680795669555664, -0.6476690769195557, -2.586808113846928e-05]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 128.32005310058594, 61.76544189453125, 64.9382095336914, 158.38278198242188, 231.08956909179688, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4065551757812, 1053.234130859375]], "actual_norm": [[376.5677185058594, 128.30807495117188, 61.711185455322266, 65.04925537109375, 158.3668975830078, 231.06753540039062, 205.61524963378906, 138.59390258789062, 33.88823699951172, 727.3442993164062, 790.3928833007812, 1053.1787109375]], "relative_norm": [[0.023809432983398438, 0.16193367540836334, 0.0532732717692852, 0.057368144392967224, 0.14921875298023224, 0.21374526619911194, 0.1981900930404663, 0.13964663445949554, 0.03523914888501167, 0.7572803497314453, 0.7120950818061829, 0.978501558303833]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 128.32005310058594, 61.76544189453125, 64.9382095336914, 158.38278198242188, 231.08956909179688, 205.5840606689453, 138.60227966308594, 33.88274002075195, 727.382568359375, 790.4065551757812, 1053.234130859375]], "actual_norm": [[376.5677185058594, 128.30807495117188, 61.711185455322266, 65.04925537109375, 158.3668975830078, 231.06753540039062, 205.61524963378906, 138.59390258789062, 33.88823699951172, 727.3442993164062, 790.3928833007812, 1053.1787109375]], "relative_norm": [[0.023809432983398438, 0.16193367540836334, 0.0532732717692852, 0.057368144392967224, 0.14921875298023224, 0.21374526619911194, 0.1981900930404663, 0.13964663445949554, 0.03523914888501167, 0.7572803497314453, 0.7120950818061829, 0.978501558303833]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Turkish lira.  Inference: The country is Turkey", "output_ids": [23734, 326, 8832, 13, 220, 758, 2202, 25, 576, 3146, 374, 17009], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 70244, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[376.9850769042969, 135.03700256347656, 62.99478530883789, 64.02460479736328, 157.65682983398438, 233.36361694335938, 202.61322021484375, 138.85902404785156, 32.77949523925781, 730.5884399414062, 790.3416748046875, 1046.9466552734375]], "actual_norm": [[376.5677185058594, 134.9921112060547, 62.94397735595703, 63.99387741088867, 157.6293182373047, 233.35350036621094, 202.66351318359375, 138.87538146972656, 32.84272384643555, 730.6378784179688, 790.3286743164062, 1047.0799560546875]], "relative_norm": [[0.023809432983398438, 0.17247308790683746, 0.054484061896800995, 0.05653737485408783, 0.14823074638843536, 0.21624305844306946, 0.19565990567207336, 0.13957291841506958, 0.034170545637607574, 0.7615209817886353, 0.7078334093093872, 0.9769312143325806]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23734, 444, 17009, 95564, 42364, 69010, 44305, 479, 19551, 13232], "first_top_logits": [22.625, 19.25, 18.25, 17.75, 16.875, 16.875, 16.5, 16.375, 16.375, 16.375], "source_correct": true, "target_correct": false, "seconds": 0.44943904085084796, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/readout", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 625.0, "target_rank": 657.5, "source_logit": 3.828125, "target_logit": 3.765625, "top_ids": [11413, 15973, 45418, 8330, 38352, 28453, 90475, 31491, 6364, 101406, 3693, 1111], "top_tokens": [" currency", "currency", ".currency", " Australia", "uenta", " Currency", "(currency", "uesto", " English", "\u8d27\u5e01", " American", "ule"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 641.0, "target_rank": 150.5, "source_logit": 3.9375, "target_logit": 5.78125, "top_ids": [11413, 15973, 96694, 28453, 7065, 3001, 3693, 53661, 5088, 45418, 26321, 34710], "top_tokens": [" currency", "currency", "onis", " Currency", "ican", "ica", " American", "cla", "urrency", ".currency", "Currency", "_currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 65.5, "target_rank": 387.0, "source_logit": 20.25, "target_logit": 16.25, "top_ids": [11413, 35602, 90475, 77186, 45418, 32488, 28453, 104142, 85466, 6323, 18762, 24357], "top_tokens": [" currency", " currencies", "(currency", ".Currency", ".currency", " euros", " Currency", "\u4eba\u6c11\u5e01", " Euros", " Japan", " coins", " overseas"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 183.0, "target_rank": 274.5, "source_logit": 14.5625, "target_logit": 13.9375, "top_ids": [11413, 35602, 7214, 104142, 3220, 47412, 15973, 34710, 6323, 28453, 32284, 17692], "top_tokens": [" currency", " currencies", " foreign", "\u4eba\u6c11\u5e01", " money", " foreigners", "currency", "_currency", " Japan", " Currency", " monetary", " dollar"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 808.5, "target_rank": 6651.0, "source_logit": 3.1875, "target_logit": 0.42578125, "top_ids": [28453, 21251, 11413, 45418, 7748, 90475, 3731, 603, 77186, 42188, 75885, 71009], "top_tokens": [" Currency", "onia", " currency", ".currency", " Indian", "(currency", " Red", "one", ".Currency", " Portuguese", " Indo", "/english"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 3027.0, "target_rank": 1900.5, "source_logit": 1.140625, "target_logit": 1.796875, "top_ids": [11413, 7179, 364, 11766, 28453, 21251, 479, 362, 18174, 356, 2932, 32], "top_tokens": [" currency", " International", " '", " Standard", " Currency", "onia", " G", " A", "oret", " C", " She", "A"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 439.5, "target_rank": 806.5, "source_logit": 15.0625, "target_logit": 14.125, "top_ids": [11413, 35602, 28453, 7576, 77186, 29970, 7214, 90475, 32488, 5837, 3146, 43895], "top_tokens": [" currency", " currencies", " Currency", " prices", ".Currency", " tourist", " foreign", "(currency", " euros", " countries", " country", " tariffs"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 173.0, "target_rank": 358.5, "source_logit": 14.375, "target_logit": 13.5, "top_ids": [11413, 35602, 7214, 6489, 5837, 37198, 6323, 7513, 5313, 7855, 94936, 3146], "top_tokens": [" currency", " currencies", " foreign", " international", " countries", " Commonwealth", " Japan", " European", " national", " British", " denomination", " country"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 5624.5, "target_rank": 7280.5, "source_logit": 0.0390625, "target_logit": -0.357421875, "top_ids": [15503, 2598, 89478, 3693, 5434, 4787, 12022, 6364, 4102, 28453, 279, 4229], "top_tokens": [" primarily", " called", "\u2026\n", " American", "\u2026\n\n", " North", " \u2026\n\n", " English", "\u00a0", " Currency", " the", " Ge"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 3743.5, "target_rank": 2776.0, "source_logit": 0.63671875, "target_logit": 1.1171875, "top_ids": [271, 4102, 4882, 5938, 3693, 11413, 1852, 7179, 479, 4787, 11888, 28453], "top_tokens": ["\n\n", "\u00a0", " South", " German", " American", " currency", " same", " International", " G", " North", " Canadian", " Currency"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 873.5, "target_rank": 2783.0, "source_logit": 13.625, "target_logit": 11.125, "top_ids": [11413, 35602, 28453, 104142, 103408, 90475, 26321, 77186, 32488, 106333, 34710, 15973], "top_tokens": [" currency", " currencies", " Currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", "(currency", "Currency", ".Currency", " euros", "\u5916\u6c47", "_currency", "currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 435.5, "target_rank": 1992.0, "source_logit": 10.125, "target_logit": 6.8125, "top_ids": [104142, 106333, 101237, 110065, 101406, 11413, 35602, 103408, 109442, 26321, 28453, 34710], "top_tokens": ["\u4eba\u6c11\u5e01", "\u5916\u6c47", "\u7f8e\u5143", "\u6c47\u7387", "\u8d27\u5e01", " currency", " currencies", "\u6b27\u5143", "\u8d22\u653f\u90e8", "Currency", " Currency", "_currency"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 7.0, "target_rank": 295.5, "source_logit": 18.375, "target_logit": 7.53125, "top_ids": [11413, 2598, 26392, 104142, 11192, 28453, 29467, 17692, 19551, 112455, 80189, 35602], "top_tokens": [" currency", " called", " Thai", "\u4eba\u6c11\u5e01", " dollars", " Currency", " Thailand", " dollar", " USD", "\u82f1\u9551", " Dollars", " currencies"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 153.0, "source_logit": 19.875, "target_logit": 9.375, "top_ids": [26392, 104142, 11413, 29467, 17692, 11192, 28453, 19551, 69185, 105906, 3639, 8453], "top_tokens": [" Thai", "\u4eba\u6c11\u5e01", " currency", " Thailand", " dollar", " dollars", " Currency", " USD", "Thai", "\u6cf0\u56fd", " United", " Chinese"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 9.0, "target_rank": 146.0, "source_logit": 32.0, "target_logit": 18.625, "top_ids": [104142, 11413, 11192, 35602, 28453, 26392, 17692, 112455, 29467, 80189, 101237, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " dollars", " currencies", " Currency", " Thai", " dollar", "\u82f1\u9551", " Thailand", " Dollars", "\u7f8e\u5143", ".Currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 4.0, "target_rank": 99.5, "source_logit": 26.125, "target_logit": 14.375, "top_ids": [104142, 26392, 11413, 29467, 28453, 35602, 105906, 101237, 101406, 17692, 69185, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " Thai", " currency", " Thailand", " Currency", " currencies", "\u6cf0\u56fd", "\u7f8e\u5143", "\u8d27\u5e01", " dollar", "Thai", ".Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 7.0, "target_rank": 524.0, "source_logit": 21.25, "target_logit": 11.25, "top_ids": [279, 2598, 26392, 3881, 14322, 12789, 29467, 33796, 7855, 220, 264, 14576], "top_tokens": [" the", " called", " Thai", " known", " Ba", " ba", " Thailand", " Camb", " British", " ", " a", " mainly"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 245.5, "source_logit": 19.5, "target_logit": 10.5625, "top_ids": [14322, 26392, 12789, 33796, 29467, 425, 7855, 4882, 16914, 663, 220, 2274], "top_tokens": [" Ba", " Thai", " ba", " Camb", " Thailand", " B", " British", " South", " Bang", " Th", " ", " US"]}], "source_token": 29467, "target_token": 22500, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Ch", "iang", " Mai", " is", " the"], "seconds": 1.158280333969742}
{"run_id": "eval-15/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.8207809925079346, "target_logp": -14.628423690795898, "log_odds": -11.807642698287964, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-1.0103756189346313, -1.810339331626892, -6.603976362384856e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-12.0103759765625, -2.6180472373962402]}}, "score_trace": {"source": [], "target": []}, "output": " Baht.  The currency used in the country containing Ch", "output_ids": [14322, 426, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 910], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [], "first_top_ids": [14322, 26392, 12789, 33796, 29467, 425, 7855, 4882, 16914, 663], "first_top_logits": [25.625, 25.375, 24.5, 21.5, 19.5, 18.375, 18.25, 18.25, 17.875, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.5418657502159476, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.8207809925079346, "target_logp": -14.628423690795898, "log_odds": -11.807642698287964, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-1.0103756189346313, -1.810339331626892, -6.603976362384856e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-12.0103759765625, -2.6180472373962402]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Baht.  The currency used in the country containing Ch", "output_ids": [14322, 426, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 910], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [14322, 26392, 12789, 33796, 29467, 425, 7855, 4882, 16914, 663], "first_top_logits": [25.625, 25.375, 24.5, 21.5, 19.5, 18.375, 18.25, 18.25, 17.875, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.5123528339900076, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -30.453563690185547, "target_logp": -9.148712158203125, "log_odds": 21.304851531982422, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-28.64264678955078, -1.8108677864074707, -4.970903682988137e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-6.5020222663879395, -2.6466903686523438]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.41842651367188, 106.6393814086914, 12.500863075256348, 7.179786205291748, 96.7859878540039, 44.98976135253906, 52.94300842285156, 113.98558044433594, 115.6634292602539, 198.828369140625, 290.6435852050781, 584.2327270507812, 705.0261840820312, 689.6458129882812]], "relative_norm": [[0.009510580450296402, 0.1342490017414093, 0.01081586629152298, 0.00632443418726325, 0.09112965315580368, 0.04161391779780388, 0.05098759010434151, 0.11490700393915176, 0.12058714777231216, 0.18153531849384308, 0.2558564245700836, 0.5828494429588318, 0.6440765857696533, 0.6600990295410156]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.41842651367188, 106.6393814086914, 12.500863075256348, 7.179786205291748, 96.7859878540039, 44.98976135253906, 52.94300842285156, 113.98558044433594, 115.6634292602539, 198.828369140625, 290.6435852050781, 584.2327270507812, 705.0261840820312, 689.6458129882812]], "relative_norm": [[0.009510580450296402, 0.1342490017414093, 0.01081586629152298, 0.00632443418726325, 0.09112965315580368, 0.04161391779780388, 0.05098759010434151, 0.11490700393915176, 0.12058714777231216, 0.18153531849384308, 0.2558564245700836, 0.5828494429588318, 0.6440765857696533, 0.6600990295410156]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnam Dong.  What is the name of the country?\n\n", "output_ids": [22500, 52383, 13, 220, 3555, 374, 279, 829, 315, 279, 3146, 1939], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 202.0288543701172, 291.9408874511719, 581.177978515625, 707.5864868164062, 685.3531494140625]], "actual_norm": [[150.41842651367188, 108.1667709350586, 13.439204216003418, 5.71180534362793, 93.80998992919922, 45.0522575378418, 56.37186050415039, 107.58375549316406, 115.25623321533203, 202.1144561767578, 291.8841247558594, 581.1685180664062, 707.5841674804688, 685.35888671875]], "relative_norm": [[0.009510580450296402, 0.1381996124982834, 0.011632923036813736, 0.0050462717190384865, 0.08821661025285721, 0.04174884036183357, 0.05442377179861069, 0.10812412202358246, 0.11991600692272186, 0.18420624732971191, 0.2574874758720398, 0.5819193124771118, 0.6469718813896179, 0.6571843028068542]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [22500, 48477, 2274, 17731, 647, 3639, 4787, 4882, 220, 547], "first_top_logits": [32.25, 25.625, 23.375, 23.125, 22.5, 22.0, 21.875, 21.25, 20.5, 20.5], "source_correct": false, "target_correct": true, "seconds": 0.4698262079618871, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.174993991851807, "target_logp": -16.285865783691406, "log_odds": -12.1108717918396, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-2.943094491958618, -1.2318248748779297, -7.509902934543788e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-14.130594253540039, -2.155271530151367]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63959503173828, 198.88633728027344, 290.6206970214844, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.3741455078125, 106.65959930419922, 12.492852210998535, 7.19500732421875, 96.72330474853516, 44.9658203125, 52.98372268676758, 113.93372344970703, 115.66740417480469, 198.9897003173828, 290.6259460449219, 584.2337646484375, 704.9953002929688, 689.6533813476562]], "relative_norm": [[0.009507780894637108, 0.1342744529247284, 0.010808935388922691, 0.006337841972708702, 0.09107062965631485, 0.04159177467226982, 0.05102680251002312, 0.11485473066568375, 0.12059129029512405, 0.18168261647224426, 0.25584089756011963, 0.5828505158424377, 0.6440483927726746, 0.6601063013076782]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63959503173828, 198.88633728027344, 290.6206970214844, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.3741455078125, 106.65959930419922, 12.492852210998535, 7.19500732421875, 96.72330474853516, 44.9658203125, 52.98372268676758, 113.93372344970703, 115.66740417480469, 198.9897003173828, 290.6259460449219, 584.2337646484375, 704.9953002929688, 689.6533813476562]], "relative_norm": [[0.009507780894637108, 0.1342744529247284, 0.010808935388922691, 0.006337841972708702, 0.09107062965631485, 0.04159177467226982, 0.05102680251002312, 0.11485473066568375, 0.12059129029512405, 0.18168261647224426, 0.25584089756011963, 0.5828505158424377, 0.6440483927726746, 0.6601063013076782]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Ch", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 910], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 202.0288543701172, 291.9408874511719, 581.177978515625, 707.5864868164062, 685.3531494140625]], "actual_norm": [[150.3741455078125, 108.1332015991211, 13.437647819519043, 5.824409484863281, 93.83672332763672, 45.004730224609375, 56.428977966308594, 107.58723449707031, 115.28466033935547, 202.0499267578125, 291.95526123046875, 581.2354736328125, 707.611328125, 685.3932495117188]], "relative_norm": [[0.009507780894637108, 0.1381567269563675, 0.011631576344370842, 0.005145755130797625, 0.08824175596237183, 0.04170479625463486, 0.05447891727089882, 0.10812762379646301, 0.11994558572769165, 0.1841474324464798, 0.2575502395629883, 0.5819863080978394, 0.6469966769218445, 0.6572172045707703]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 26392, 33796, 12789, 6633, 663, 29467, 220, 13, 198], "first_top_logits": [26.625, 23.75, 20.75, 20.375, 20.375, 20.0, 19.875, 19.375, 19.0, 18.625], "source_correct": false, "target_correct": false, "seconds": 0.43609958281740546, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -8.418582916259766, "target_logp": -39.85406494140625, "log_odds": -31.435482025146484, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-7.000962257385254, -1.4175469875335693, -7.331102824537084e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-37.19627380371094, -2.6577913761138916]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.38111877441406, 106.63458251953125, 12.504682540893555, 7.182333946228027, 96.72093963623047, 45.01103210449219, 52.97040939331055, 113.90581512451172, 115.64752197265625, 198.9262237548828, 290.61883544921875, 584.1867065429688, 704.91796875, 689.59375]], "relative_norm": [[0.009508221410214901, 0.1342429667711258, 0.01081917155534029, 0.006326678674668074, 0.09106840938329697, 0.041633594781160355, 0.05101398006081581, 0.11482658982276917, 0.12057056277990341, 0.1816246509552002, 0.2558346390724182, 0.5828035473823547, 0.643977701663971, 0.6600492000579834]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.38111877441406, 106.63458251953125, 12.504682540893555, 7.182333946228027, 96.72093963623047, 45.01103210449219, 52.97040939331055, 113.90581512451172, 115.64752197265625, 198.9262237548828, 290.61883544921875, 584.1867065429688, 704.91796875, 689.59375]], "relative_norm": [[0.009508221410214901, 0.1342429667711258, 0.01081917155534029, 0.006326678674668074, 0.09106840938329697, 0.041633594781160355, 0.05101398006081581, 0.11482658982276917, 0.12057056277990341, 0.1816246509552002, 0.2558346390724182, 0.5828035473823547, 0.643977701663971, 0.6600492000579834]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thailand Baht. The currency used in the country containing the", "output_ids": [29467, 14322, 426, 13, 576, 11413, 1483, 304, 279, 3146, 8482, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 202.0288543701172, 291.9408874511719, 581.177978515625, 707.5864868164062, 685.3531494140625]], "actual_norm": [[150.38111877441406, 108.17135620117188, 13.433222770690918, 5.70799446105957, 93.81109619140625, 45.04265213012695, 56.3668327331543, 107.58522033691406, 115.21853637695312, 202.02146911621094, 292.0749206542969, 581.0875854492188, 707.688232421875, 685.368408203125]], "relative_norm": [[0.009508221410214901, 0.13820546865463257, 0.011627745814621449, 0.0050429049879312515, 0.08821765333414078, 0.04173993691802025, 0.05441891774535179, 0.10812559723854065, 0.11987678706645966, 0.1841215044260025, 0.2576557993888855, 0.5818382501602173, 0.6470670104026794, 0.6571934223175049]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29467, 26392, 105906, 59016, 4434, 69185, 663, 20539, 120695, 25178], "first_top_logits": [35.5, 28.5, 25.0, 25.0, 21.75, 20.125, 19.375, 19.25, 18.5, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.46161687513813376, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.842848300933838, "target_logp": -13.541963577270508, "log_odds": -11.69911527633667, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.028994396328926086, -1.8137743473052979, -7.950943836476654e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-11.0289945602417, -2.5129687786102295]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.6343994140625, 12.458035469055176, 7.032834053039551, 96.73512268066406, 44.96455383300781, 52.96586608886719, 113.92234802246094, 115.63958740234375, 198.88633728027344, 290.6206359863281, 584.1944580078125, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.17039489746094, 106.63553619384766, 12.477580070495605, 7.195374011993408, 96.79389953613281, 44.95014190673828, 52.941001892089844, 113.93073272705078, 115.60640716552734, 198.91348266601562, 290.61590576171875, 584.2683715820312, 705.071533203125, 689.6331176757812]], "relative_norm": [[0.009494897909462452, 0.1342441588640213, 0.010795721784234047, 0.006338165141642094, 0.0911371037364006, 0.04157727211713791, 0.050985660403966904, 0.11485171318054199, 0.12052769213914871, 0.181613028049469, 0.2558320462703705, 0.5828850269317627, 0.6441180109977722, 0.6600868701934814]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.6343994140625, 12.458035469055176, 7.032834053039551, 96.73512268066406, 44.96455383300781, 52.96586608886719, 113.92234802246094, 115.63958740234375, 198.88633728027344, 290.6206359863281, 584.1944580078125, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.17039489746094, 106.63553619384766, 12.477580070495605, 7.195374011993408, 96.79389953613281, 44.95014190673828, 52.941001892089844, 113.93073272705078, 115.60640716552734, 198.91348266601562, 290.61590576171875, 584.2683715820312, 705.071533203125, 689.6331176757812]], "relative_norm": [[0.009494897909462452, 0.1342441588640213, 0.010795721784234047, 0.006338165141642094, 0.0911371037364006, 0.04157727211713791, 0.050985660403966904, 0.11485171318054199, 0.12052769213914871, 0.181613028049469, 0.2558320462703705, 0.5828850269317627, 0.6441180109977722, 0.6600868701934814]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  What is the currency used in the", "output_ids": [26392, 14322, 426, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15623474121094, 13.393844604492188, 5.649867057800293, 93.83389282226562, 44.991607666015625, 56.41873550415039, 107.56111907958984, 115.24628448486328, 202.0288543701172, 291.94085693359375, 581.177978515625, 707.58642578125, 685.3530883789062]], "actual_norm": [[150.17039489746094, 108.14505004882812, 13.38752555847168, 5.760975360870361, 93.87027740478516, 44.99168014526367, 56.37632369995117, 107.64241027832031, 115.25211334228516, 202.05508422851562, 291.9609680175781, 581.203125, 707.642333984375, 685.3493041992188]], "relative_norm": [[0.009494897909462452, 0.13817186653614044, 0.01158819068223238, 0.005089712329208851, 0.0882733091711998, 0.04169270396232605, 0.05442808195948601, 0.10818307101726532, 0.11991172283887863, 0.18415214121341705, 0.2575552761554718, 0.5819539427757263, 0.6470250487327576, 0.6571750640869141]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 663, 7855, 4882, 29467, 33796, 14322, 2274, 22450, 13], "first_top_logits": [25.0, 20.125, 19.875, 19.25, 18.5, 18.375, 18.0, 17.375, 17.25, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.48468424985185266, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.532689332962036, "target_logp": -12.247567176818848, "log_odds": -8.714877843856812, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-1.824502944946289, -1.7081222534179688, -6.41325386823155e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-9.574502944946289, -2.6730642318725586]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458036422729492, 7.032835006713867, 96.73513793945312, 44.964561462402344, 52.96586990356445, 113.92236328125, 115.63959503173828, 198.8863525390625, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.35951232910156, 106.64459228515625, 12.529890060424805, 7.179718971252441, 96.62397766113281, 44.93440628051758, 52.89712905883789, 114.00179290771484, 115.6473388671875, 198.9072265625, 290.7057800292969, 584.199951171875, 705.0591430664062, 689.7269897460938]], "relative_norm": [[0.00950685515999794, 0.1342555582523346, 0.010840981267392635, 0.006324375048279762, 0.09097710996866226, 0.04156271740794182, 0.05094340816140175, 0.11492335051298141, 0.12057036906480789, 0.18160730600357056, 0.2559111714363098, 0.5828167796134949, 0.6441066861152649, 0.6601767539978027]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458036422729492, 7.032835006713867, 96.73513793945312, 44.964561462402344, 52.96586990356445, 113.92236328125, 115.63959503173828, 198.8863525390625, 290.62066650390625, 584.1945190429688, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.35951232910156, 106.64459228515625, 12.529890060424805, 7.179718971252441, 96.62397766113281, 44.93440628051758, 52.89712905883789, 114.00179290771484, 115.6473388671875, 198.9072265625, 290.7057800292969, 584.199951171875, 705.0591430664062, 689.7269897460938]], "relative_norm": [[0.00950685515999794, 0.1342555582523346, 0.010840981267392635, 0.006324375048279762, 0.09097710996866226, 0.04156271740794182, 0.05094340816140175, 0.11492335051298141, 0.12057036906480789, 0.18160730600357056, 0.2559111714363098, 0.5828167796134949, 0.6441066861152649, 0.6601767539978027]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Baht.  The currency used in the country containing Ch", "output_ids": [14322, 426, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 910], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390808105469, 44.991607666015625, 56.418739318847656, 107.56112670898438, 115.24629974365234, 202.0288543701172, 291.94091796875, 581.177978515625, 707.5865478515625, 685.3531494140625]], "actual_norm": [[150.35951232910156, 108.20101928710938, 13.457687377929688, 5.8210015296936035, 93.90157318115234, 44.9215087890625, 56.43233871459961, 107.66671752929688, 115.27252960205078, 202.0962677001953, 291.9695739746094, 581.1797485351562, 707.5507202148438, 685.3253784179688]], "relative_norm": [[0.00950685515999794, 0.13824336230754852, 0.011648922227323055, 0.005142744164913893, 0.08830273896455765, 0.0416276790201664, 0.05448216199874878, 0.10820750147104263, 0.11993296444416046, 0.18418966233730316, 0.2575628459453583, 0.5819305181503296, 0.6469413042068481, 0.6571521162986755]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [14322, 26392, 12789, 425, 13, 120695, 1532, 68884, 29467, 2274], "first_top_logits": [21.375, 20.0, 19.375, 19.125, 18.375, 16.125, 16.125, 15.875, 15.75, 15.6875], "source_correct": true, "target_correct": false, "seconds": 0.46259737480431795, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-15/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-15", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Chiang Mai", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Chiang Mai is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9804859161376953, "target_logp": -13.802651405334473, "log_odds": -11.822165489196777, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.06236810237169266, -1.9180172681808472, -0.00010048838157672435]}, "target": {"ids": [48477, 72736], "log_probs": [-10.937368392944336, -2.865283250808716]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 106.63440704345703, 12.458035469055176, 7.032834053039551, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1944580078125, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.2759552001953, 106.62873077392578, 12.477922439575195, 7.155562877655029, 96.75682830810547, 45.01399612426758, 53.00661087036133, 113.92506408691406, 115.64107513427734, 198.88626098632812, 290.572509765625, 584.2186279296875, 705.0012817382812, 689.6150512695312]], "relative_norm": [[0.009501571767032146, 0.13423559069633484, 0.010796017944812775, 0.006303096655756235, 0.09110219776630402, 0.04163633659482002, 0.051048845052719116, 0.11484599858522415, 0.12056383490562439, 0.18158817291259766, 0.25579383969306946, 0.5828353762626648, 0.6440538167953491, 0.6600695848464966]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 106.63440704345703, 12.458035469055176, 7.032834053039551, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 198.88633728027344, 290.62066650390625, 584.1944580078125, 705.0107421875, 689.6095581054688]], "actual_norm": [[150.2759552001953, 106.62873077392578, 12.477922439575195, 7.155562877655029, 96.75682830810547, 45.01399612426758, 53.00661087036133, 113.92506408691406, 115.64107513427734, 198.88626098632812, 290.572509765625, 584.2186279296875, 705.0012817382812, 689.6150512695312]], "relative_norm": [[0.009501571767032146, 0.13423559069633484, 0.010796017944812775, 0.006303096655756235, 0.09110219776630402, 0.04163633659482002, 0.051048845052719116, 0.11484599858522415, 0.12056383490562439, 0.18158817291259766, 0.25579383969306946, 0.5828353762626648, 0.6440538167953491, 0.6600695848464966]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  In the country containing Chiang Mai", "output_ids": [26392, 14322, 426, 13, 220, 758, 279, 3146, 8482, 910, 27224, 42256], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 910, 27224, 42256, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 108.15623474121094, 13.393844604492188, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 202.0288543701172, 291.94085693359375, 581.177978515625, 707.58642578125, 685.3531494140625]], "actual_norm": [[150.2759552001953, 108.12553405761719, 13.481757164001465, 5.769858360290527, 93.80221557617188, 45.04542922973633, 56.40296936035156, 107.53182220458984, 115.25983428955078, 202.01748657226562, 291.9294128417969, 581.0820922851562, 707.5211181640625, 685.388916015625]], "relative_norm": [[0.009501571767032146, 0.13814692199230194, 0.011669756844639778, 0.005097560118883848, 0.08820930123329163, 0.041742511093616486, 0.05445380508899689, 0.10807193070650101, 0.11991975456476212, 0.1841178685426712, 0.2575274407863617, 0.581832766532898, 0.6469141840934753, 0.6572130918502808]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 29467, 33796, 14322, 12789, 663, 17936, 7855, 220, 13], "first_top_logits": [25.625, 22.0, 21.125, 20.5, 20.375, 19.25, 18.25, 18.0, 17.5, 17.375], "source_correct": true, "target_correct": false, "seconds": 0.46871150005608797, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/readout", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 1033.5, "target_rank": 757.0, "source_logit": 2.96875, "target_logit": 3.328125, "top_ids": [15973, 8330, 11413, 45418, 7065, 3001, 38352, 6364, 3693, 31491, 7748, 7513], "top_tokens": ["currency", " Australia", " currency", ".currency", "ican", "ica", "uenta", " English", " American", "uesto", " Indian", " European"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 1432.0, "target_rank": 228.5, "source_logit": 2.609375, "target_logit": 4.9375, "top_ids": [11413, 15973, 96694, 3001, 53661, 3693, 28453, 5088, 7065, 11496, 21295, 28130], "top_tokens": [" currency", "currency", "onis", "ica", "cla", " American", " Currency", "urrency", "ican", "ika", "ollar", "ija"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 110.0, "target_rank": 506.0, "source_logit": 18.25, "target_logit": 15.375, "top_ids": [11413, 35602, 24357, 6489, 8330, 7214, 13369, 7513, 36445, 7855, 3146, 85466], "top_tokens": [" currency", " currencies", " overseas", " international", " Australia", " foreign", " Australian", " European", " internationally", " British", " country", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 451.0, "target_rank": 585.5, "source_logit": 12.5625, "target_logit": 12.125, "top_ids": [11413, 7214, 13224, 35602, 7855, 47412, 3220, 6489, 37198, 13369, 8330, 19078], "top_tokens": [" currency", " foreign", " Britain", " currencies", " British", " foreigners", " money", " international", " Commonwealth", " Australian", " Australia", " Foreign"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 4688.0, "target_rank": 12887.0, "source_logit": 0.283203125, "target_logit": -1.359375, "top_ids": [21251, 28453, 7513, 11413, 3731, 7748, 56537, 603, 3776, 12751, 55964, 5737], "top_tokens": ["onia", " Currency", " European", " currency", " Red", " Indian", " _______,", "one", " One", " Roman", "\u2011", " Group"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 8791.5, "target_rank": 4154.0, "source_logit": -0.76953125, "target_logit": 0.443359375, "top_ids": [11766, 11413, 364, 7179, 479, 356, 362, 28453, 18174, 467, 6489, 5297], "top_tokens": [" Standard", " currency", " '", " International", " G", " C", " A", " Currency", "oret", " W", " international", " standard"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 2258.5, "target_rank": 2660.5, "source_logit": 12.0625, "target_logit": 11.8125, "top_ids": [11413, 35602, 13867, 3146, 5313, 7214, 30176, 28453, 60261, 5837, 29973, 2205], "top_tokens": [" currency", " currencies", " county", " country", " national", " foreign", " counties", " Currency", " tariff", " countries", " licence", " local"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 434.0, "target_rank": 1019.5, "source_logit": 13.5625, "target_logit": 12.1875, "top_ids": [11413, 35602, 37198, 7214, 7855, 7513, 6364, 6489, 15973, 105492, 94936, 5837], "top_tokens": [" currency", " currencies", " Commonwealth", " foreign", " British", " European", " English", " international", "currency", "\u4e2d\u534e\u4eba\u6c11", " denomination", " countries"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 11866.5, "target_rank": 9017.0, "source_logit": -1.640625, "target_logit": -1.125, "top_ids": [89478, 3693, 4787, 5434, 4229, 4102, 279, 220, 271, 6364, 46427, 11], "top_tokens": ["\u2026\n", " American", " North", "\u2026\n\n", " Ge", "\u00a0", " the", " ", "\n\n", " English", "/is", ","]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 5299.5, "target_rank": 2434.0, "source_logit": -0.07080078125, "target_logit": 1.15625, "top_ids": [4102, 1852, 271, 3693, 4882, 4787, 3639, 5938, 479, 7179, 362, 6489], "top_tokens": ["\u00a0", " same", "\n\n", " American", " South", " North", " United", " German", " G", " International", " A", " international"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 2141.0, "target_rank": 4896.0, "source_logit": 10.1875, "target_logit": 8.25, "top_ids": [11413, 35602, 104142, 28453, 103408, 32488, 113883, 85466, 26321, 106333, 112455, 90475], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u6b27\u5143", " euros", "\u672c\u56fd", " Euros", "Currency", "\u5916\u6c47", "\u82f1\u9551", "(currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 320.5, "target_rank": 2106.5, "source_logit": 9.75, "target_logit": 5.78125, "top_ids": [104142, 116866, 106333, 101237, 109442, 112397, 103408, 113883, 110065, 105579, 106127, 114141], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u7f8e\u5143", "\u8d22\u653f\u90e8", "\u56fd\u5bb6\u6807\u51c6", "\u6b27\u5143", "\u672c\u56fd", "\u6c47\u7387", "\u6fb3\u5927\u5229\u4e9a", "\u6b27\u76df", "\u81ea\u7531\u8d38\u6613"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 15.0, "target_rank": 602.5, "source_logit": 18.0, "target_logit": 6.3125, "top_ids": [11413, 11192, 104142, 26392, 17692, 112455, 28453, 19551, 32488, 2598, 7855, 80189], "top_tokens": [" currency", " dollars", "\u4eba\u6c11\u5e01", " Thai", " dollar", "\u82f1\u9551", " Currency", " USD", " euros", " called", " British", " Dollars"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 267.0, "source_logit": 20.375, "target_logit": 7.78125, "top_ids": [26392, 104142, 11413, 17692, 29467, 19551, 11192, 28453, 3639, 7855, 105906, 42364], "top_tokens": [" Thai", "\u4eba\u6c11\u5e01", " currency", " dollar", " Thailand", " USD", " dollars", " Currency", " United", " British", "\u6cf0\u56fd", " Dollar"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 17.0, "target_rank": 273.0, "source_logit": 30.25, "target_logit": 15.875, "top_ids": [104142, 11413, 11192, 112455, 35602, 28453, 17692, 101237, 19551, 32488, 80189, 103408], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " dollars", "\u82f1\u9551", " currencies", " Currency", " dollar", "\u7f8e\u5143", " USD", " euros", " Dollars", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 4.0, "target_rank": 153.0, "source_logit": 27.375, "target_logit": 13.0625, "top_ids": [104142, 26392, 11413, 29467, 35602, 112455, 28453, 101237, 105906, 19551, 17692, 110065], "top_tokens": ["\u4eba\u6c11\u5e01", " Thai", " currency", " Thailand", " currencies", "\u82f1\u9551", " Currency", "\u7f8e\u5143", "\u6cf0\u56fd", " USD", " dollar", "\u6c47\u7387"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 5.0, "target_rank": 1789.0, "source_logit": 22.625, "target_logit": 8.625, "top_ids": [279, 26392, 7855, 2598, 29467, 3881, 220, 14322, 264, 1083, 12789, 663], "top_tokens": [" the", " Thai", " British", " called", " Thailand", " known", " ", " Ba", " a", " also", " ba", " Th"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 823.5, "source_logit": 24.0, "target_logit": 8.375, "top_ids": [26392, 14322, 29467, 12789, 7855, 69185, 131053, 663, 12922, 220, 59016, 17936], "top_tokens": [" Thai", " Ba", " Thailand", " ba", " British", "Thai", " Th\u00e1i", " Th", " island", " ", " Bangkok", " thai"]}], "source_token": 29467, "target_token": 22500, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Ph", "uk", "et", " is", " the"], "seconds": 1.1630733329802752}
{"run_id": "eval-16/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.6315181255340576, "target_logp": -18.442306518554688, "log_odds": -16.81078839302063, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.020808136090636253, -1.610611081123352, -9.893881360767409e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-15.895808219909668, -2.5464978218078613]}}, "score_trace": {"source": [], "target": []}, "output": " Thai Baht.  In the country containing Phuket", "output_ids": [26392, 14322, 426, 13, 220, 758, 279, 3146, 8482, 2350, 3101, 295], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [], "first_top_ids": [26392, 14322, 29467, 12789, 7855, 69185, 131053, 663, 12922, 220], "first_top_logits": [29.0, 24.625, 24.0, 22.375, 21.25, 20.25, 19.625, 19.5, 19.25, 19.25], "source_correct": true, "target_correct": false, "seconds": 0.5091494577936828, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.6315181255340576, "target_logp": -18.442306518554688, "log_odds": -16.81078839302063, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.020808136090636253, -1.610611081123352, -9.893881360767409e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-15.895808219909668, -2.5464978218078613]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  In the country containing Phuket", "output_ids": [26392, 14322, 426, 13, 220, 758, 279, 3146, 8482, 2350, 3101, 295], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 14322, 29467, 12789, 7855, 69185, 131053, 663, 12922, 220], "first_top_logits": [29.0, 24.625, 24.0, 22.375, 21.25, 20.25, 19.625, 19.5, 19.25, 19.25], "source_correct": true, "target_correct": false, "seconds": 0.45189041597768664, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -31.637042999267578, "target_logp": -10.507806777954102, "log_odds": 21.129236221313477, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-30.12574005126953, -1.5112336874008179, -6.794698856538162e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-7.750741004943848, -2.7570652961730957]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833984375, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.41842651367188, 106.6393814086914, 12.500863075256348, 7.179786205291748, 96.7859878540039, 44.98976135253906, 52.94300842285156, 113.98558044433594, 115.6634292602539, 74.24031829833984, 889.486572265625, 708.7406616210938, 798.4351806640625, 823.3330078125]], "relative_norm": [[0.009510580450296402, 0.1342490017414093, 0.01081586629152298, 0.00632443418726325, 0.09112965315580368, 0.04161391779780388, 0.05098759010434151, 0.11490700393915176, 0.12058714777231216, 0.06152571737766266, 0.6824147701263428, 0.729669451713562, 0.6974666118621826, 0.7858139276504517]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833984375, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.41842651367188, 106.6393814086914, 12.500863075256348, 7.179786205291748, 96.7859878540039, 44.98976135253906, 52.94300842285156, 113.98558044433594, 115.6634292602539, 74.24031829833984, 889.486572265625, 708.7406616210938, 798.4351806640625, 823.3330078125]], "relative_norm": [[0.009510580450296402, 0.1342490017414093, 0.01081586629152298, 0.00632443418726325, 0.09112965315580368, 0.04161391779780388, 0.05098759010434151, 0.11490700393915176, 0.12058714777231216, 0.06152571737766266, 0.6824147701263428, 0.729669451713562, 0.6974666118621826, 0.7858139276504517]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnam Dong.  What is the currency used in the country", "output_ids": [22500, 52383, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 70.31204223632812, 894.0381469726562, 704.869140625, 805.779541015625, 823.9639892578125]], "actual_norm": [[150.41842651367188, 108.1667709350586, 13.439204216003418, 5.71180534362793, 93.80998992919922, 45.0522575378418, 56.37186050415039, 107.58375549316406, 115.25623321533203, 70.31327056884766, 894.0408935546875, 704.860595703125, 805.7300415039062, 824.0132446289062]], "relative_norm": [[0.009510580450296402, 0.1381996124982834, 0.011632923036813736, 0.0050462717190384865, 0.08821661025285721, 0.04174884036183357, 0.05442377179861069, 0.10812412202358246, 0.11991600692272186, 0.058188170194625854, 0.6865124702453613, 0.7265682816505432, 0.7041419744491577, 0.7866831421852112]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [22500, 48477, 17731, 2274, 3639, 647, 94054, 17692, 84359, 4787], "first_top_logits": [31.75, 24.0, 22.75, 21.875, 21.25, 21.25, 20.375, 20.125, 19.875, 19.75], "source_correct": false, "target_correct": true, "seconds": 0.4464984158985317, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.8103513717651367, "target_logp": -16.416385650634766, "log_odds": -12.606034278869629, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-2.8206896781921387, -0.9895832538604736, -7.843663479434326e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-14.32068920135498, -2.095695734024048]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63959503173828, 74.23534393310547, 889.4833984375, 708.7484741210938, 798.5064697265625, 823.3109741210938]], "actual_norm": [[150.3741455078125, 106.65959930419922, 12.492852210998535, 7.19500732421875, 96.72330474853516, 44.9658203125, 52.98372268676758, 113.93372344970703, 115.66740417480469, 74.165283203125, 889.5037841796875, 708.7330932617188, 798.5182495117188, 823.1987915039062]], "relative_norm": [[0.009507780894637108, 0.1342744529247284, 0.010808935388922691, 0.006337841972708702, 0.09107062965631485, 0.04159177467226982, 0.05102680251002312, 0.11485473066568375, 0.12059129029512405, 0.06146353483200073, 0.6824279427528381, 0.7296616435050964, 0.6975391507148743, 0.7856858372688293]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63959503173828, 74.23534393310547, 889.4833984375, 708.7484741210938, 798.5064697265625, 823.3109741210938]], "actual_norm": [[150.3741455078125, 106.65959930419922, 12.492852210998535, 7.19500732421875, 96.72330474853516, 44.9658203125, 52.98372268676758, 113.93372344970703, 115.66740417480469, 74.165283203125, 889.5037841796875, 708.7330932617188, 798.5182495117188, 823.1987915039062]], "relative_norm": [[0.009507780894637108, 0.1342744529247284, 0.010808935388922691, 0.006337841972708702, 0.09107062965631485, 0.04159177467226982, 0.05102680251002312, 0.11485473066568375, 0.12059129029512405, 0.06146353483200073, 0.6824279427528381, 0.7296616435050964, 0.6975391507148743, 0.7856858372688293]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Ph", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 2350], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 70.31204986572266, 894.0382080078125, 704.869140625, 805.779541015625, 823.9639892578125]], "actual_norm": [[150.3741455078125, 108.1332015991211, 13.437647819519043, 5.824409484863281, 93.83672332763672, 45.004730224609375, 56.428977966308594, 107.58723449707031, 115.28466033935547, 70.216552734375, 894.1385498046875, 704.8234252929688, 805.83349609375, 823.8681030273438]], "relative_norm": [[0.009507780894637108, 0.1381567269563675, 0.011631576344370842, 0.005145755130797625, 0.08824175596237183, 0.04170479625463486, 0.05447891727089882, 0.10812762379646301, 0.11994558572769165, 0.0581081286072731, 0.6865874528884888, 0.7265299558639526, 0.704232394695282, 0.7865445613861084]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 26392, 29467, 6633, 220, 12922, 12122, 198, 663, 120695], "first_top_logits": [27.375, 24.5, 21.5, 20.875, 20.25, 20.125, 19.0, 19.0, 18.875, 18.625], "source_correct": false, "target_correct": false, "seconds": 0.46033899998292327, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -9.161624908447266, "target_logp": -41.02408981323242, "log_odds": -31.862464904785156, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-8.000349998474121, -1.1611802577972412, -9.405170567333698e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-38.59410095214844, -2.429987668991089]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833984375, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.38111877441406, 106.63458251953125, 12.504682540893555, 7.182333946228027, 96.72093963623047, 45.01103210449219, 52.97040939331055, 113.90581512451172, 115.64752197265625, 74.27244567871094, 889.4644165039062, 708.7481079101562, 798.642822265625, 823.4408569335938]], "relative_norm": [[0.009508221410214901, 0.1342429667711258, 0.01081917155534029, 0.006326678674668074, 0.09106840938329697, 0.041633594781160355, 0.05101398006081581, 0.11482658982276917, 0.12057056277990341, 0.061552345752716064, 0.6823977828025818, 0.729677140712738, 0.6976479887962341, 0.7859168648719788]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458035469055176, 7.032834529876709, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833984375, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.38111877441406, 106.63458251953125, 12.504682540893555, 7.182333946228027, 96.72093963623047, 45.01103210449219, 52.97040939331055, 113.90581512451172, 115.64752197265625, 74.27244567871094, 889.4644165039062, 708.7481079101562, 798.642822265625, 823.4408569335938]], "relative_norm": [[0.009508221410214901, 0.1342429667711258, 0.01081917155534029, 0.006326678674668074, 0.09106840938329697, 0.041633594781160355, 0.05101398006081581, 0.11482658982276917, 0.12057056277990341, 0.061552345752716064, 0.6823977828025818, 0.729677140712738, 0.6976479887962341, 0.7859168648719788]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thailand Thai Baht.  What is the currency used in", "output_ids": [29467, 26392, 14322, 426, 13, 220, 3555, 374, 279, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 70.31204223632812, 894.0381469726562, 704.869140625, 805.779541015625, 823.9639892578125]], "actual_norm": [[150.38111877441406, 108.17135620117188, 13.433222770690918, 5.70799446105957, 93.81109619140625, 45.04265213012695, 56.3668327331543, 107.58522033691406, 115.21853637695312, 70.28653717041016, 894.03466796875, 704.8391723632812, 805.8085327148438, 823.9100952148438]], "relative_norm": [[0.009508221410214901, 0.13820546865463257, 0.011627745814621449, 0.0050429049879312515, 0.08821765333414078, 0.04173993691802025, 0.05441891774535179, 0.10812559723854065, 0.11987678706645966, 0.05816604569554329, 0.6865077018737793, 0.7265462279319763, 0.7042105793952942, 0.7865846753120422]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29467, 26392, 105906, 59016, 4434, 20539, 25178, 663, 2205, 120695], "first_top_logits": [34.5, 26.625, 22.875, 22.5, 21.125, 19.625, 19.125, 18.75, 18.5, 17.875], "source_correct": true, "target_correct": false, "seconds": 0.5009220829233527, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.5366008281707764, "target_logp": -15.328158378601074, "log_odds": -13.791557550430298, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.018895387649536133, -1.5176036357879639, -0.00010179955279454589]}, "target": {"ids": [48477, 72736], "log_probs": [-12.893895149230957, -2.434263229370117]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.6343994140625, 12.458035469055176, 7.032834053039551, 96.73512268066406, 44.96455383300781, 52.96586608886719, 113.92234802246094, 115.63958740234375, 74.23534393310547, 889.4833374023438, 708.7483520507812, 798.5064697265625, 823.3109130859375]], "actual_norm": [[150.17039489746094, 106.63553619384766, 12.477580070495605, 7.195374011993408, 96.79389953613281, 44.95014190673828, 52.941001892089844, 113.93073272705078, 115.60640716552734, 74.23442840576172, 889.5319213867188, 708.7486572265625, 798.6098022460938, 823.191650390625]], "relative_norm": [[0.009494897909462452, 0.1342441588640213, 0.010795721784234047, 0.006338165141642094, 0.0911371037364006, 0.04157727211713791, 0.050985660403966904, 0.11485171318054199, 0.12052769213914871, 0.061520837247371674, 0.6824495792388916, 0.729677677154541, 0.6976191401481628, 0.785679042339325]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.6343994140625, 12.458035469055176, 7.032834053039551, 96.73512268066406, 44.96455383300781, 52.96586608886719, 113.92234802246094, 115.63958740234375, 74.23534393310547, 889.4833374023438, 708.7483520507812, 798.5064697265625, 823.3109130859375]], "actual_norm": [[150.17039489746094, 106.63553619384766, 12.477580070495605, 7.195374011993408, 96.79389953613281, 44.95014190673828, 52.941001892089844, 113.93073272705078, 115.60640716552734, 74.23442840576172, 889.5319213867188, 708.7486572265625, 798.6098022460938, 823.191650390625]], "relative_norm": [[0.009494897909462452, 0.1342441588640213, 0.010795721784234047, 0.006338165141642094, 0.0911371037364006, 0.04157727211713791, 0.050985660403966904, 0.11485171318054199, 0.12052769213914871, 0.061520837247371674, 0.6824495792388916, 0.729677677154541, 0.6976191401481628, 0.785679042339325]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  What is the currency used in the", "output_ids": [26392, 14322, 426, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15623474121094, 13.393844604492188, 5.649867057800293, 93.83389282226562, 44.991607666015625, 56.41873550415039, 107.56111907958984, 115.24628448486328, 70.3120346069336, 894.0380859375, 704.8690795898438, 805.779541015625, 823.9639892578125]], "actual_norm": [[150.17039489746094, 108.14505004882812, 13.38752555847168, 5.760975360870361, 93.87027740478516, 44.99168014526367, 56.37632369995117, 107.64241027832031, 115.25211334228516, 70.2873306274414, 894.0621948242188, 704.8123779296875, 805.8269653320312, 823.9384155273438]], "relative_norm": [[0.009494897909462452, 0.13817186653614044, 0.01158819068223238, 0.005089712329208851, 0.0882733091711998, 0.04169270396232605, 0.05442808195948601, 0.10818307101726532, 0.11991172283887863, 0.05816670134663582, 0.6865288615226746, 0.7265185713768005, 0.7042267322540283, 0.7866117358207703]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 7855, 29467, 663, 17672, 19461, 105906, 4882, 6424, 35564], "first_top_logits": [27.5, 22.875, 21.875, 20.75, 20.5, 20.0, 19.375, 18.5, 18.375, 18.125], "source_correct": true, "target_correct": false, "seconds": 0.4657732080668211, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.034217596054077, "target_logp": -13.15814208984375, "log_odds": -11.123924493789673, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.6155670285224915, -1.4185595512390137, -9.107174992095679e-05]}, "target": {"ids": [48477, 72736], "log_probs": [-10.303067207336426, -2.855074644088745]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458036422729492, 7.032835006713867, 96.73513793945312, 44.964561462402344, 52.96586990356445, 113.92236328125, 115.63959503173828, 74.23534393310547, 889.4834594726562, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.35951232910156, 106.64459228515625, 12.529890060424805, 7.179718971252441, 96.62397766113281, 44.93440628051758, 52.89712905883789, 114.00179290771484, 115.6473388671875, 74.33010864257812, 889.517822265625, 708.7367553710938, 798.477783203125, 823.3508911132812]], "relative_norm": [[0.00950685515999794, 0.1342555582523346, 0.010840981267392635, 0.006324375048279762, 0.09097710996866226, 0.04156271740794182, 0.05094340816140175, 0.11492335051298141, 0.12057036906480789, 0.061600130051374435, 0.6824387311935425, 0.7296653985977173, 0.6975038051605225, 0.7858310341835022]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 106.63440704345703, 12.458036422729492, 7.032835006713867, 96.73513793945312, 44.964561462402344, 52.96586990356445, 113.92236328125, 115.63959503173828, 74.23534393310547, 889.4834594726562, 708.7484130859375, 798.5065307617188, 823.3109741210938]], "actual_norm": [[150.35951232910156, 106.64459228515625, 12.529890060424805, 7.179718971252441, 96.62397766113281, 44.93440628051758, 52.89712905883789, 114.00179290771484, 115.6473388671875, 74.33010864257812, 889.517822265625, 708.7367553710938, 798.477783203125, 823.3508911132812]], "relative_norm": [[0.00950685515999794, 0.1342555582523346, 0.010840981267392635, 0.006324375048279762, 0.09097710996866226, 0.04156271740794182, 0.05094340816140175, 0.11492335051298141, 0.12057036906480789, 0.061600130051374435, 0.6824387311935425, 0.7296653985977173, 0.6975038051605225, 0.7858310341835022]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  In the country containing Phuket", "output_ids": [26392, 14322, 426, 13, 220, 758, 279, 3146, 8482, 2350, 3101, 295], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38565063476562, 108.15624237060547, 13.393845558166504, 5.649867534637451, 93.83390808105469, 44.991607666015625, 56.418739318847656, 107.56112670898438, 115.24629974365234, 70.31204223632812, 894.0381469726562, 704.8692016601562, 805.7796020507812, 823.9640502929688]], "actual_norm": [[150.35951232910156, 108.20101928710938, 13.457687377929688, 5.8210015296936035, 93.90157318115234, 44.9215087890625, 56.43233871459961, 107.66671752929688, 115.27252960205078, 70.29105377197266, 894.0151977539062, 704.9303588867188, 805.7605590820312, 824.0078735351562]], "relative_norm": [[0.00950685515999794, 0.13824336230754852, 0.011648922227323055, 0.005142744164913893, 0.08830273896455765, 0.0416276790201664, 0.05448216199874878, 0.10820750147104263, 0.11993296444416046, 0.058169782161712646, 0.6864927411079407, 0.7266402244567871, 0.7041686773300171, 0.7866780161857605]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 14322, 13, 19461, 12922, 29467, 120695, 12789, 425, 13369], "first_top_logits": [20.5, 19.125, 18.625, 18.0, 17.75, 17.25, 17.125, 17.125, 16.75, 16.5], "source_correct": true, "target_correct": false, "seconds": 0.44927862472832203, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-16/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-16", "split": "eval", "country": "Thailand", "currency": "Thai baht", "aliases": ["baht", "THB"], "city": "Phuket", "target_country": "Vietnam", "target_currency": "Vietnamese dong", "target_aliases": ["dong", "VND"], "prompt": "Fact: The currency used in the country containing Phuket is the", "source_continuation": " Thai baht", "target_continuation": " Vietnamese dong"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.7694976329803467, "target_logp": -14.940479278564453, "log_odds": -13.170981645584106, "token_scores": {"source": {"ids": [26392, 12789, 426], "log_probs": [-0.054841116070747375, -1.7145181894302368, -0.0001382732152706012]}, "target": {"ids": [48477, 72736], "log_probs": [-11.992341041564941, -2.9481379985809326]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 106.63440704345703, 12.458035469055176, 7.032834053039551, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833374023438, 708.7484130859375, 798.5064697265625, 823.3109130859375]], "actual_norm": [[150.2759552001953, 106.62873077392578, 12.477922439575195, 7.155562877655029, 96.75682830810547, 45.01399612426758, 53.00661087036133, 113.92506408691406, 115.64107513427734, 74.31964111328125, 889.3788452148438, 708.7386474609375, 798.5191650390625, 823.298828125]], "relative_norm": [[0.009501571767032146, 0.13423559069633484, 0.010796017944812775, 0.006303096655756235, 0.09110219776630402, 0.04163633659482002, 0.051048845052719116, 0.11484599858522415, 0.12056383490562439, 0.061591457575559616, 0.6823320984840393, 0.7296673655509949, 0.6975399255752563, 0.7857813239097595]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 106.63440704345703, 12.458035469055176, 7.032834053039551, 96.7351303100586, 44.96455764770508, 52.96586990356445, 113.92235565185547, 115.63958740234375, 74.23534393310547, 889.4833374023438, 708.7484130859375, 798.5064697265625, 823.3109130859375]], "actual_norm": [[150.2759552001953, 106.62873077392578, 12.477922439575195, 7.155562877655029, 96.75682830810547, 45.01399612426758, 53.00661087036133, 113.92506408691406, 115.64107513427734, 74.31964111328125, 889.3788452148438, 708.7386474609375, 798.5191650390625, 823.298828125]], "relative_norm": [[0.009501571767032146, 0.13423559069633484, 0.010796017944812775, 0.006303096655756235, 0.09110219776630402, 0.04163633659482002, 0.051048845052719116, 0.11484599858522415, 0.12056383490562439, 0.061591457575559616, 0.6823320984840393, 0.7296673655509949, 0.6975399255752563, 0.7857813239097595]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Thai Baht.  In the country containing Phuket", "output_ids": [26392, 14322, 426, 13, 220, 758, 279, 3146, 8482, 2350, 3101, 295], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 2350, 3101, 295, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[150.38563537597656, 108.15623474121094, 13.393844604492188, 5.649867534637451, 93.83390045166016, 44.991607666015625, 56.418739318847656, 107.56111907958984, 115.24629211425781, 70.3120346069336, 894.0380859375, 704.8690795898438, 805.779541015625, 823.9639282226562]], "actual_norm": [[150.2759552001953, 108.12553405761719, 13.481757164001465, 5.769858360290527, 93.80221557617188, 45.04542922973633, 56.40296936035156, 107.53182220458984, 115.25983428955078, 70.28862762451172, 894.0847778320312, 704.8975830078125, 805.7598876953125, 823.9468994140625]], "relative_norm": [[0.009501571767032146, 0.13814692199230194, 0.011669756844639778, 0.005097560118883848, 0.08820930123329163, 0.041742511093616486, 0.05445380508899689, 0.10807193070650101, 0.11991975456476212, 0.058167774230241776, 0.6865462064743042, 0.7266064286231995, 0.7041680812835693, 0.7866197824478149]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [26392, 29467, 663, 12922, 14322, 17936, 7855, 13369, 12789, 19551], "first_top_logits": [25.5, 22.5, 19.25, 19.25, 18.875, 18.875, 18.625, 18.625, 18.375, 17.875], "source_correct": true, "target_correct": false, "seconds": 0.4244593749754131, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/readout", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 278.5, "target_rank": 877.5, "source_logit": 4.28125, "target_logit": 2.921875, "top_ids": [11413, 15973, 8330, 45418, 38352, 28453, 6364, 3693, 31491, 56181, 7065, 13369], "top_tokens": [" currency", "currency", " Australia", ".currency", "uenta", " Currency", " English", " American", "uesto", " Philippine", "ican", " Australian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 123.5, "target_rank": 537.0, "source_logit": 5.90625, "target_logit": 4.0625, "top_ids": [11413, 15973, 28453, 96694, 45418, 34065, 5088, 26321, 77186, 3001, 7065, 21295], "top_tokens": [" currency", "currency", " Currency", "onis", ".currency", "_money", "urrency", "Currency", ".Currency", "ica", "ican", "ollar"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 601.0, "target_rank": 933.0, "source_logit": 15.0, "target_logit": 13.75, "top_ids": [11413, 35602, 90475, 45418, 32488, 77186, 6489, 12463, 8330, 104142, 24357, 7214], "top_tokens": [" currency", " currencies", "(currency", ".currency", " euros", ".Currency", " international", " colour", " Australia", "\u4eba\u6c11\u5e01", " overseas", " foreign"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 453.5, "target_rank": 421.0, "source_logit": 13.6875, "target_logit": 13.875, "top_ids": [11413, 35602, 3220, 7214, 104142, 34710, 32284, 15973, 78253, 28453, 17692, 47412], "top_tokens": [" currency", " currencies", " money", " foreign", "\u4eba\u6c11\u5e01", "_currency", " monetary", "currency", " treasury", " Currency", " dollar", " foreigners"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 6783.5, "target_rank": 1848.5, "source_logit": -0.099609375, "target_logit": 1.65625, "top_ids": [21251, 3731, 55964, 28453, 3776, 56537, 12751, 5737, 7748, 32671, 603, 11413], "top_tokens": ["onia", " Red", "\u2011", " Currency", " One", " _______,", " Roman", " Group", " Indian", " ______", "one", " currency"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 2608.5, "target_rank": 1204.0, "source_logit": 1.1328125, "target_logit": 2.234375, "top_ids": [11413, 7179, 364, 11766, 18174, 479, 28453, 2932, 32, 362, 21251, 105469], "top_tokens": [" currency", " International", " '", " Standard", "oret", " G", " Currency", " She", "A", " A", "onia", "\u79cd\u7c7b"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1528.5, "target_rank": 2061.0, "source_logit": 12.6875, "target_logit": 12.25, "top_ids": [11413, 35602, 3146, 5313, 5837, 7576, 7214, 3033, 13867, 28453, 6489, 24357], "top_tokens": [" currency", " currencies", " country", " national", " countries", " prices", " foreign", " government", " county", " Currency", " international", " overseas"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 669.5, "target_rank": 449.0, "source_logit": 12.75, "target_logit": 13.3125, "top_ids": [11413, 35602, 37198, 94936, 7214, 15973, 6489, 7513, 17672, 110065, 105492, 5837], "top_tokens": [" currency", " currencies", " Commonwealth", " denomination", " foreign", "currency", " international", " European", " euro", "\u6c47\u7387", "\u4e2d\u534e\u4eba\u6c11", " countries"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 4948.0, "target_rank": 9808.5, "source_logit": 0.06982421875, "target_logit": -1.0703125, "top_ids": [2598, 3693, 1083, 279, 4787, 4229, 89478, 4102, 15503, 111012, 5434, 220], "top_tokens": [" called", " American", " also", " the", " North", " Ge", "\u2026\n", "\u00a0", " primarily", "\u4e16\u754c\u4e0a\u6700", "\u2026\n\n", " "]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 788.0, "target_rank": 3767.5, "source_logit": 2.859375, "target_logit": 0.5703125, "top_ids": [1852, 4102, 3693, 3639, 5938, 271, 5055, 4787, 479, 7179, 362, 220], "top_tokens": [" same", "\u00a0", " American", " United", " German", "\n\n", " National", " North", " G", " International", " A", " "]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 3099.5, "target_rank": 3951.5, "source_logit": 10.6875, "target_logit": 10.125, "top_ids": [11413, 35602, 104142, 103408, 28453, 113883, 32488, 26321, 90475, 106333, 115160, 77186], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " Currency", "\u672c\u56fd", " euros", "Currency", "(currency", "\u5916\u6c47", "\u5176\u4ed6\u56fd\u5bb6", ".Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 1542.5, "target_rank": 2547.0, "source_logit": 7.125, "target_logit": 6.0625, "top_ids": [104142, 116866, 112397, 109442, 113883, 101237, 106333, 110065, 99599, 101406, 102120, 104531], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u8d22\u653f\u90e8", "\u672c\u56fd", "\u7f8e\u5143", "\u5916\u6c47", "\u6c47\u7387", "\u56fd\u5bb6", "\u8d27\u5e01", "\u8d22\u653f", "\u4eba\u6c11\u653f\u5e9c"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 22.0, "target_rank": 9943.5, "source_logit": 17.5, "target_logit": 1.0703125, "top_ids": [104142, 48477, 11413, 11192, 28453, 2598, 17692, 19551, 35602, 80189, 77186, 15973], "top_tokens": ["\u4eba\u6c11\u5e01", " Vietnamese", " currency", " dollars", " Currency", " called", " dollar", " USD", " currencies", " Dollars", ".Currency", "currency"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 11737.5, "source_logit": 22.125, "target_logit": 0.3984375, "top_ids": [48477, 104142, 11413, 94054, 22500, 28453, 17692, 128324, 19551, 101406, 106676, 11192], "top_tokens": [" Vietnamese", "\u4eba\u6c11\u5e01", " currency", "Viet", " Vietnam", " Currency", " dollar", " Vi\u1ec7t", " USD", "\u8d27\u5e01", "\u8d8a\u5357", " dollars"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 32.5, "target_rank": 3378.5, "source_logit": 28.25, "target_logit": 9.1875, "top_ids": [104142, 11413, 35602, 11192, 28453, 48477, 17692, 101406, 80189, 32488, 19551, 18762], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " currencies", " dollars", " Currency", " Vietnamese", " dollar", "\u8d27\u5e01", " Dollars", " euros", " USD", " coins"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 8.0, "target_rank": 4407.5, "source_logit": 29.25, "target_logit": 5.65625, "top_ids": [104142, 48477, 11413, 101406, 94054, 28453, 35602, 22500, 106676, 17692, 77186, 101237], "top_tokens": ["\u4eba\u6c11\u5e01", " Vietnamese", " currency", "\u8d27\u5e01", "Viet", " Currency", " currencies", " Vietnam", "\u8d8a\u5357", " dollar", ".Currency", "\u7f8e\u5143"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 14.0, "target_rank": 23866.5, "source_logit": 19.875, "target_logit": 2.390625, "top_ids": [279, 2598, 48477, 3881, 8585, 220, 6941, 647, 4882, 4787, 128329, 2274], "top_tokens": [" the", " called", " Vietnamese", " known", " French", " ", " named", " V", " South", " North", " \u0111\u1ed3ng", " US"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 39684.0, "source_logit": 21.25, "target_logit": -0.9921875, "top_ids": [48477, 52383, 22500, 647, 4787, 8585, 128329, 72736, 4882, 84359, 17731, 220], "top_tokens": [" Vietnamese", " Dong", " Vietnam", " V", " North", " French", " \u0111\u1ed3ng", " dong", " South", " viet", " Viet", " "]}], "source_token": 22500, "target_token": 27602, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " H", "anoi", " is", " the"], "seconds": 1.1661920826882124}
{"run_id": "eval-17/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.54378080368042, "target_logp": -19.598691940307617, "log_odds": -18.054911136627197, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.019860224798321724, -1.5239205360412598]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-16.394779205322266, -1.511826515197754, -0.003196489531546831, -1.6888904571533203]}}, "score_trace": {"source": [], "target": []}, "output": " Vietnamese \u0111\u1ed3ng.\n\nFact: The currency used in the country containing", "output_ids": [48477, 128329, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [], "first_top_ids": [48477, 52383, 22500, 647, 4787, 128329, 8585, 72736, 4882, 17731], "first_top_logits": [26.75, 21.625, 21.25, 20.75, 20.25, 19.875, 19.875, 18.75, 18.75, 18.625], "source_correct": false, "target_correct": false, "seconds": 0.47630904195830226, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.54378080368042, "target_logp": -19.598691940307617, "log_odds": -18.054911136627197, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.019860224798321724, -1.5239205360412598]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-16.394779205322266, -1.511826515197754, -0.003196489531546831, -1.6888904571533203]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese \u0111\u1ed3ng.\n\nFact: The currency used in the country containing", "output_ids": [48477, 128329, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 52383, 22500, 647, 4787, 128329, 8585, 72736, 4882, 17731], "first_top_logits": [26.75, 21.625, 21.25, 20.75, 20.25, 19.875, 19.875, 18.75, 18.75, 18.625], "source_correct": false, "target_correct": false, "seconds": 0.46332912519574165, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -51.21551513671875, "target_logp": -16.106142044067383, "log_odds": 35.10937309265137, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-50.625003814697266, -0.5905123949050903]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-12.87500286102295, -1.458631992340088, -0.0033524997998028994, -1.7691551446914673]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.30000305175781, 128.2342529296875, 77.5366439819336, 66.31688690185547, 134.4031982421875, 188.37588500976562, 110.73413848876953, 51.26554489135742, 101.1332778930664, 483.7435607910156, 892.2213134765625, 1130.408447265625]], "actual_norm": [[10.597234725952148, 100.29817962646484, 128.26837158203125, 77.5342788696289, 66.3321304321289, 134.39785766601562, 188.38539123535156, 110.72358703613281, 51.27536392211914, 101.1191635131836, 483.7217102050781, 892.245361328125, 1130.4241943359375]], "relative_norm": [[0.0006700365920551121, 0.1265832483768463, 0.11072993278503418, 0.06837891787290573, 0.06250042468309402, 0.1243225559592247, 0.18158243596553802, 0.11156462877988815, 0.05331939458847046, 0.09973670542240143, 0.5163153409957886, 0.7433775067329407, 1.0062928199768066]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 100.0409164428711, 484.9576110839844, 892.4804077148438, 1131.9100341796875]], "actual_norm": [[10.597234725952148, 101.41195678710938, 128.58316040039062, 75.77538299560547, 65.2911605834961, 134.56015014648438, 189.3873291015625, 108.84941864013672, 52.927764892578125, 100.07404327392578, 484.9642639160156, 892.4036254882812, 1131.8712158203125]], "relative_norm": [[0.0006700365920551121, 0.12766817212104797, 0.11125138401985168, 0.0667480081319809, 0.06147543713450432, 0.12446331977844238, 0.1823924332857132, 0.10972931981086731, 0.0551808662712574, 0.09842893481254578, 0.5173524618148804, 0.7387127876281738, 1.0057908296585083]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Poland's z\u0142oty.  What is the country's", "output_ids": [27602, 594, 58068, 354, 88, 13, 220, 3555, 374, 279, 3146, 594], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 102.72530364990234, 487.2315673828125, 894.5565185546875, 1121.373291015625]], "actual_norm": [[10.597234725952148, 99.88351440429688, 131.52182006835938, 75.39046478271484, 64.74720001220703, 137.087158203125, 193.6033172607422, 107.1884536743164, 51.027610778808594, 102.71929931640625, 487.2507019042969, 894.4859619140625, 1121.53759765625]], "relative_norm": [[0.0006700365920551121, 0.12761647999286652, 0.11384477466344833, 0.06660604476928711, 0.060886677354574203, 0.12703536450862885, 0.18691280484199524, 0.10772683471441269, 0.05309063941240311, 0.10129342973232269, 0.5178130269050598, 0.7448974847793579, 1.001784324645996]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27602, 31984, 1471, 112020, 3651, 57719, 14658, 44029, 444, 393], "first_top_logits": [37.5, 24.5, 22.375, 21.0, 20.625, 19.875, 19.125, 18.625, 18.25, 18.0], "source_correct": false, "target_correct": true, "seconds": 0.45116424979642034, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.969379425048828, "target_logp": -22.68900489807129, "log_odds": -15.719625473022461, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-5.882033348083496, -1.087345838546753]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-19.632116317749023, -1.4740684032440186, -0.0020988595206290483, -1.5807225704193115]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.30001068115234, 128.2342529296875, 77.5366439819336, 66.31688690185547, 134.4031982421875, 188.37588500976562, 110.73413848876953, 51.265541076660156, 101.1332778930664, 483.74359130859375, 892.2213134765625, 1130.408447265625]], "actual_norm": [[10.574127197265625, 100.32646179199219, 128.19505310058594, 77.60885620117188, 66.34637451171875, 134.3799285888672, 188.37269592285156, 110.74887084960938, 51.26676940917969, 101.11064147949219, 483.7439880371094, 892.245849609375, 1130.390380859375]], "relative_norm": [[0.0006685755797661841, 0.12661893665790558, 0.11066664010286331, 0.06844468414783478, 0.06251384317874908, 0.12430597096681595, 0.18157020211219788, 0.11159010231494904, 0.05331045761704445, 0.0997283011674881, 0.5163390636444092, 0.7433778643608093, 1.0062627792358398]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.87972640991211, 100.0409164428711, 484.9576416015625, 892.4804077148438, 1131.91015625]], "actual_norm": [[10.574127197265625, 101.4311752319336, 128.6035614013672, 75.83547973632812, 65.24454498291016, 134.52688598632812, 189.44252014160156, 108.841552734375, 52.88261032104492, 100.07173156738281, 484.9473571777344, 892.4949340820312, 1131.9754638671875]], "relative_norm": [[0.0006685755797661841, 0.1276923567056656, 0.1112690344452858, 0.06680094450712204, 0.06143154576420784, 0.12443255633115768, 0.18244558572769165, 0.10972139239311218, 0.05513378605246544, 0.09842666238546371, 0.5173344612121582, 0.738788366317749, 1.0058834552764893]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing H", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 472], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 102.72530364990234, 487.2315673828125, 894.5565795898438, 1121.3734130859375]], "actual_norm": [[10.574127197265625, 99.91732788085938, 131.5339813232422, 75.3414077758789, 64.7527847290039, 137.03257751464844, 193.58447265625, 107.20710754394531, 51.06242752075195, 102.73589324951172, 487.2375793457031, 894.515380859375, 1121.37841796875]], "relative_norm": [[0.0006685755797661841, 0.12765967845916748, 0.11385530233383179, 0.0665627047419548, 0.060891930013895035, 0.12698478996753693, 0.18689461052417755, 0.10774558782577515, 0.05312686413526535, 0.10130979865789413, 0.5177990794181824, 0.7449219822883606, 1.0016422271728516]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 48477, 6633, 659, 134597, 220, 12122, 13, 198, 1140], "first_top_logits": [28.375, 22.5, 22.0, 20.625, 20.375, 20.0, 19.875, 19.75, 19.5, 19.125], "source_correct": false, "target_correct": false, "seconds": 0.5056327497586608, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -7.8751325607299805, "target_logp": -48.04973220825195, "log_odds": -40.17459964752197, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-6.376831531524658, -1.4983010292053223]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-44.50181579589844, -1.6698856353759766, -0.0026774294674396515, -1.8753540515899658]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.30000305175781, 128.2342529296875, 77.5366439819336, 66.31688690185547, 134.4031982421875, 188.37588500976562, 110.73413848876953, 51.26554489135742, 101.1332778930664, 483.7435607910156, 892.2213134765625, 1130.408447265625]], "actual_norm": [[10.58515739440918, 100.30022430419922, 128.25364685058594, 77.50763702392578, 66.35902404785156, 134.43214416503906, 188.33689880371094, 110.72844696044922, 51.2784309387207, 101.17347717285156, 483.7406921386719, 892.2355346679688, 1130.3477783203125]], "relative_norm": [[0.0006692729657515883, 0.12658582627773285, 0.11071722209453583, 0.06835541874170303, 0.06252576410770416, 0.1243542730808258, 0.18153569102287292, 0.11156952381134033, 0.05332258343696594, 0.09979027509689331, 0.5163355469703674, 0.7433692812919617, 1.0062248706817627]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 100.0409164428711, 484.9576110839844, 892.4804077148438, 1131.9100341796875]], "actual_norm": [[10.58515739440918, 101.44721984863281, 128.60348510742188, 75.79740142822266, 65.27684020996094, 134.5218963623047, 189.4027862548828, 108.89777374267578, 52.902557373046875, 100.06068420410156, 485.03216552734375, 892.6038818359375, 1131.7882080078125]], "relative_norm": [[0.0006692729657515883, 0.12771256268024445, 0.11126896739006042, 0.06676740199327469, 0.061461951583623886, 0.12442793697118759, 0.18240731954574585, 0.10977806895971298, 0.05515458434820175, 0.0984157919883728, 0.5174248814582825, 0.7388785481452942, 1.0057170391082764]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnam Dong.  The currency used in the country containing H", "output_ids": [22500, 52383, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 472], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 102.72530364990234, 487.2315673828125, 894.5565185546875, 1121.373291015625]], "actual_norm": [[10.58515739440918, 99.8964614868164, 131.49807739257812, 75.40767669677734, 64.7639389038086, 137.0272216796875, 193.5645751953125, 107.19935607910156, 51.015113830566406, 102.71464538574219, 487.2699279785156, 894.674072265625, 1121.2999267578125]], "relative_norm": [[0.0006692729657515883, 0.1276330202817917, 0.11382422596216202, 0.06662125140428543, 0.060902420431375504, 0.12697981297969818, 0.18687540292739868, 0.10773779451847076, 0.053077638149261475, 0.10128884017467499, 0.5178334712982178, 0.7450541853904724, 1.0015721321105957]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [22500, 48477, 17731, 220, 94054, 647, 84359, 128324, 106676, 609], "first_top_logits": [31.875, 25.5, 22.625, 19.625, 19.5, 19.125, 19.125, 19.0, 18.875, 18.5], "source_correct": true, "target_correct": false, "seconds": 0.42529429169371724, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.129940152168274, "target_logp": -17.274429321289062, "log_odds": -16.14448916912079, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.06016799062490463, -1.0697721242904663]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-14.128195762634277, -1.5684388875961304, -0.0026745761279016733, -1.5751206874847412]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.29999542236328, 128.23423767089844, 77.5366439819336, 66.31688690185547, 134.4031982421875, 188.37588500976562, 110.734130859375, 51.265541076660156, 101.1332778930664, 483.7435302734375, 892.2212524414062, 1130.408447265625]], "actual_norm": [[10.552916526794434, 100.28431701660156, 128.23606872558594, 77.57894897460938, 66.30506134033203, 134.45379638671875, 188.33755493164062, 110.77755737304688, 51.29621887207031, 101.25811767578125, 483.6868591308594, 892.2695922851562, 1130.3162841796875]], "relative_norm": [[0.0006672344752587378, 0.12656575441360474, 0.11070205271244049, 0.06841830909252167, 0.062474917620420456, 0.12437430024147034, 0.18153633177280426, 0.1116190031170845, 0.053341079503297806, 0.09987375885248184, 0.516278088092804, 0.7433976531028748, 1.0061967372894287]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.60308837890625, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.40640258789062, 108.9019775390625, 52.87971878051758, 100.04090881347656, 484.95758056640625, 892.4803466796875, 1131.9100341796875]], "actual_norm": [[10.552916526794434, 101.45048522949219, 128.61045837402344, 75.87791442871094, 65.17778778076172, 134.5554962158203, 189.37551879882812, 108.9405517578125, 52.91693878173828, 100.11705017089844, 484.96014404296875, 892.4638061523438, 1131.85986328125]], "relative_norm": [[0.0006672344752587378, 0.12771667540073395, 0.11127500236034393, 0.0668383240699768, 0.06136868894100189, 0.12445901334285736, 0.1823810487985611, 0.10982119292020798, 0.05516957864165306, 0.09847123175859451, 0.517348051071167, 0.738762617111206, 1.0057806968688965]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese \u0111\u1ed3ng.  The currency used in the country containing H", "output_ids": [48477, 128329, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 472], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90184783935547, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321228027344, 51.04629898071289, 102.72529602050781, 487.2315368652344, 894.5564575195312, 1121.373291015625]], "actual_norm": [[10.552916526794434, 99.857666015625, 131.54396057128906, 75.45318603515625, 64.69916534423828, 137.00015258789062, 193.49868774414062, 107.2196044921875, 51.0920524597168, 102.82056427001953, 487.2012939453125, 894.61376953125, 1121.2191162109375]], "relative_norm": [[0.0006672344752587378, 0.12758345901966095, 0.11386394500732422, 0.06666145473718643, 0.06084150820970535, 0.12695473432540894, 0.18681178987026215, 0.10775814205408096, 0.053157687187194824, 0.10139328986406326, 0.5177605152130127, 0.7450039386749268, 1.001499891281128]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 8585, 19461, 17672, 22500, 220, 30, 84359, 106676, 2097], "first_top_logits": [25.125, 22.0, 19.875, 19.75, 19.625, 19.375, 18.375, 18.125, 17.5, 17.25], "source_correct": false, "target_correct": false, "seconds": 0.4428802910260856, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.809462308883667, "target_logp": -16.14514923095703, "log_odds": -14.335686922073364, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.21538475155830383, -1.5940775871276855]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-12.364068031311035, -1.6285037994384766, -0.00323320715688169, -2.1493446826934814]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.30001068115234, 128.2342529296875, 77.5366439819336, 66.31689453125, 134.40321350097656, 188.37588500976562, 110.73414611816406, 51.26554489135742, 101.1332778930664, 483.74359130859375, 892.2213745117188, 1130.408447265625]], "actual_norm": [[10.546387672424316, 100.31355285644531, 128.26181030273438, 77.55504608154297, 66.25439453125, 134.43759155273438, 188.4160919189453, 110.83870697021484, 51.2963981628418, 101.10889434814453, 483.7303161621094, 892.2213745117188, 1130.4239501953125]], "relative_norm": [[0.0006668216665275395, 0.1266026496887207, 0.11072427034378052, 0.06839723140001297, 0.062427178025245667, 0.12435930967330933, 0.181612029671669, 0.11168061941862106, 0.05334126576781273, 0.09972657263278961, 0.516324520111084, 0.7433574795722961, 1.0062925815582275]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.23081970214844, 134.5460968017578, 189.40643310546875, 108.90198516845703, 52.879722595214844, 100.0409164428711, 484.9576416015625, 892.4804077148438, 1131.91015625]], "actual_norm": [[10.546387672424316, 101.42137145996094, 128.65260314941406, 75.8039321899414, 65.20276641845703, 134.55233764648438, 189.3370361328125, 108.94488525390625, 52.9619140625, 100.13687133789062, 485.00335693359375, 892.4097900390625, 1131.8939208984375]], "relative_norm": [[0.0006668216665275395, 0.12768001854419708, 0.11131146550178528, 0.06677315384149551, 0.06139220669865608, 0.12445609271526337, 0.182343989610672, 0.10982555896043777, 0.0552164688706398, 0.09849072992801666, 0.5173941850662231, 0.7387178540229797, 1.0058109760284424]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese \u0111\u1ed3ng.\n\nFact: The currency used in the country containing", "output_ids": [48477, 128329, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 102.72530364990234, 487.2315979003906, 894.5565795898438, 1121.3734130859375]], "actual_norm": [[10.546387672424316, 99.89518737792969, 131.57762145996094, 75.3753890991211, 64.6564712524414, 137.0599822998047, 193.5853271484375, 107.31085205078125, 51.074981689453125, 102.81329345703125, 487.3122863769531, 894.454345703125, 1121.366943359375]], "relative_norm": [[0.0006668216665275395, 0.12763139605522156, 0.11389307677745819, 0.06659272313117981, 0.06080136075615883, 0.12701018154621124, 0.1868954449892044, 0.1078498512506485, 0.053139928728342056, 0.10138612240552902, 0.5178784728050232, 0.744871199131012, 1.0016319751739502]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 17731, 13, 22500, 94054, 892, 128324, 128329, 106676, 19461], "first_top_logits": [22.25, 20.5, 18.375, 18.125, 17.625, 17.375, 17.125, 16.75, 16.75, 16.625], "source_correct": false, "target_correct": false, "seconds": 0.49212995870038867, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-17/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-17", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Hanoi", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Hanoi is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2490824460983276, "target_logp": -21.24925422668457, "log_odds": -20.000171780586243, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.01568680629134178, -1.2333956956863403]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-18.57709312438965, -1.3437808752059937, -0.002651748713105917, -1.3257302045822144]}}, "score_trace": {"source": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 100.29999542236328, 128.23423767089844, 77.5366439819336, 66.31687927246094, 134.40318298339844, 188.37588500976562, 110.73413848876953, 51.265541076660156, 101.1332778930664, 483.7435302734375, 892.2212524414062, 1130.408447265625]], "actual_norm": [[10.564362525939941, 100.3789291381836, 128.25827026367188, 77.54061889648438, 66.31253814697266, 134.38331604003906, 188.3276824951172, 110.73149871826172, 51.258750915527344, 101.18450927734375, 483.6975402832031, 892.177734375, 1130.353515625]], "relative_norm": [[0.0006679581711068749, 0.12668515741825104, 0.11072121560573578, 0.06838450580835342, 0.06248196214437485, 0.12430910766124725, 0.1815268099308014, 0.11157259345054626, 0.05330211669206619, 0.09980116039514542, 0.5162895321846008, 0.7433211207389832, 1.0062298774719238]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.9019775390625, 52.879722595214844, 100.04090881347656, 484.9576110839844, 892.4804077148438, 1131.9100341796875]], "actual_norm": [[10.564362525939941, 101.44377899169922, 128.5926971435547, 75.87361145019531, 65.23687744140625, 134.54446411132812, 189.32424926757812, 108.93573760986328, 52.85751724243164, 100.05536651611328, 484.9053649902344, 892.4602661132812, 1131.8260498046875]], "relative_norm": [[0.0006679581711068749, 0.12770822644233704, 0.11125963926315308, 0.06683453172445297, 0.06142432615160942, 0.12444881349802017, 0.18233168125152588, 0.10981634259223938, 0.05510762706398964, 0.09841056168079376, 0.5172896385192871, 0.7387596368789673, 1.0057506561279297]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese \u0111\u1ed3ng.\n\nFact: The currency used in the country containing", "output_ids": [48477, 128329, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 472, 72703, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[10.557646751403809, 99.90184783935547, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 102.72530364990234, 487.2315368652344, 894.5565185546875, 1121.373291015625]], "actual_norm": [[10.564362525939941, 99.9435806274414, 131.49911499023438, 75.4498519897461, 64.73917388916016, 136.97665405273438, 193.52032470703125, 107.17867279052734, 51.03163146972656, 102.8017578125, 487.2209167480469, 894.5053100585938, 1121.2645263671875]], "relative_norm": [[0.0006679581711068749, 0.12769322097301483, 0.11382512748241425, 0.06665851175785065, 0.06087912991642952, 0.12693296372890472, 0.18683268129825592, 0.10771700739860535, 0.05309482291340828, 0.10137474536895752, 0.5177813768386841, 0.744913637638092, 1.0015404224395752]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 22500, 17731, 94054, 84359, 647, 220, 128324, 68884, 17692], "first_top_logits": [25.125, 19.75, 19.5, 19.125, 18.25, 17.875, 17.0, 17.0, 16.375, 15.9375], "source_correct": false, "target_correct": false, "seconds": 0.49924566596746445, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/readout", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 363.0, "target_rank": 504.5, "source_logit": 4.34375, "target_logit": 3.921875, "top_ids": [15973, 11413, 8330, 45418, 38352, 28453, 6364, 49029, 90475, 7065, 3001, 7748], "top_tokens": ["currency", " currency", " Australia", ".currency", "uenta", " Currency", " English", "ctica", "(currency", "ican", "ica", " Indian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 93.0, "target_rank": 268.0, "source_logit": 6.25, "target_logit": 4.875, "top_ids": [11413, 15973, 28453, 96694, 3001, 7065, 45418, 53661, 26321, 5088, 42189, 34710], "top_tokens": [" currency", "currency", " Currency", "onis", "ica", "ican", ".currency", "cla", "Currency", "urrency", "etz", "_currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 362.0, "target_rank": 883.0, "source_logit": 16.5, "target_logit": 14.0, "top_ids": [11413, 35602, 90475, 45418, 77186, 32488, 28453, 6489, 104142, 34274, 24357, 6323], "top_tokens": [" currency", " currencies", "(currency", ".currency", ".Currency", " euros", " Currency", " international", "\u4eba\u6c11\u5e01", " Caribbean", " overseas", " Japan"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 276.5, "target_rank": 202.0, "source_logit": 14.625, "target_logit": 15.125, "top_ids": [11413, 7214, 35602, 47412, 3220, 104142, 6489, 6323, 34710, 28453, 15973, 5837], "top_tokens": [" currency", " foreign", " currencies", " foreigners", " money", "\u4eba\u6c11\u5e01", " international", " Japan", "_currency", " Currency", "currency", " countries"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 5099.0, "target_rank": 1712.5, "source_logit": 0.3515625, "target_logit": 1.890625, "top_ids": [21251, 28453, 7748, 3731, 11413, 3776, 76231, 55964, 42188, 7513, 19833, 5737], "top_tokens": ["onia", " Currency", " Indian", " Red", " currency", " One", " Maritime", "\u2011", " Portuguese", " European", " Amb", " Group"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1425.0, "target_rank": 679.0, "source_logit": 2.0625, "target_logit": 3.109375, "top_ids": [7179, 11413, 364, 11766, 479, 6489, 18174, 28453, 362, 467, 547, 3639], "top_tokens": [" International", " currency", " '", " Standard", " G", " international", "oret", " Currency", " A", " W", " U", " United"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 856.5, "target_rank": 1739.5, "source_logit": 14.0, "target_logit": 12.875, "top_ids": [11413, 35602, 3146, 5837, 28453, 13867, 7576, 5313, 7214, 30176, 11192, 24357], "top_tokens": [" currency", " currencies", " country", " countries", " Currency", " county", " prices", " national", " foreign", " counties", " dollars", " overseas"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 291.0, "target_rank": 321.5, "source_logit": 14.5, "target_logit": 14.375, "top_ids": [11413, 35602, 7214, 6489, 105492, 5837, 7513, 110065, 6323, 37198, 94936, 17672], "top_tokens": [" currency", " currencies", " foreign", " international", "\u4e2d\u534e\u4eba\u6c11", " countries", " European", "\u6c47\u7387", " Japan", " Commonwealth", " denomination", " euro"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 4279.5, "target_rank": 9311.0, "source_logit": 0.453125, "target_logit": -0.85546875, "top_ids": [3693, 4787, 4229, 89478, 5434, 15503, 279, 6364, 2598, 46427, 4102, 12022], "top_tokens": [" American", " North", " Ge", "\u2026\n", "\u2026\n\n", " primarily", " the", " English", " called", "/is", "\u00a0", " \u2026\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 995.0, "target_rank": 4866.0, "source_logit": 2.5625, "target_logit": 0.09228515625, "top_ids": [1852, 4102, 3693, 271, 479, 7179, 5938, 3639, 4787, 4882, 220, 5055], "top_tokens": [" same", "\u00a0", " American", "\n\n", " G", " International", " German", " United", " North", " South", " ", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 2083.0, "target_rank": 3133.5, "source_logit": 10.8125, "target_logit": 9.8125, "top_ids": [11413, 35602, 104142, 28453, 103408, 113883, 26321, 90475, 106333, 77186, 32488, 101406], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u6b27\u5143", "\u672c\u56fd", "Currency", "(currency", "\u5916\u6c47", ".Currency", " euros", "\u8d27\u5e01"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 622.5, "target_rank": 2806.5, "source_logit": 9.625, "target_logit": 6.125, "top_ids": [104142, 116866, 106333, 101237, 110065, 109442, 103408, 112397, 101406, 113883, 11413, 35602], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u7f8e\u5143", "\u6c47\u7387", "\u8d22\u653f\u90e8", "\u6b27\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u8d27\u5e01", "\u672c\u56fd", " currency", " currencies"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 8.0, "target_rank": 19780.5, "source_logit": 20.125, "target_logit": -0.5625, "top_ids": [48477, 11413, 104142, 11192, 19551, 17692, 28453, 22500, 80189, 94054, 42364, 35602], "top_tokens": [" Vietnamese", " currency", "\u4eba\u6c11\u5e01", " dollars", " USD", " dollar", " Currency", " Vietnam", " Dollars", "Viet", " Dollar", " currencies"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 26407.0, "source_logit": 24.0, "target_logit": -1.625, "top_ids": [48477, 104142, 22500, 94054, 11413, 106676, 128324, 19551, 17692, 28453, 42364, 17731], "top_tokens": [" Vietnamese", "\u4eba\u6c11\u5e01", " Vietnam", "Viet", " currency", "\u8d8a\u5357", " Vi\u1ec7t", " USD", " dollar", " Currency", " Dollar", " Viet"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 20.0, "target_rank": 6526.5, "source_logit": 31.625, "target_logit": 7.34375, "top_ids": [104142, 11413, 48477, 11192, 35602, 28453, 17692, 19551, 101406, 101237, 80189, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Vietnamese", " dollars", " currencies", " Currency", " dollar", " USD", "\u8d27\u5e01", "\u7f8e\u5143", " Dollars", ".Currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 4.0, "target_rank": 10988.5, "source_logit": 32.0, "target_logit": 3.65625, "top_ids": [48477, 104142, 94054, 22500, 106676, 11413, 28453, 35602, 17692, 101406, 19551, 101237], "top_tokens": [" Vietnamese", "\u4eba\u6c11\u5e01", "Viet", " Vietnam", "\u8d8a\u5357", " currency", " Currency", " currencies", " dollar", "\u8d27\u5e01", " USD", "\u7f8e\u5143"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 10.0, "target_rank": 45324.0, "source_logit": 20.625, "target_logit": 0.2734375, "top_ids": [279, 48477, 647, 3881, 2598, 2274, 128329, 220, 4882, 22500, 8585, 4787], "top_tokens": [" the", " Vietnamese", " V", " known", " called", " US", " \u0111\u1ed3ng", " ", " South", " Vietnam", " French", " North"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 55389.5, "source_logit": 23.375, "target_logit": -2.328125, "top_ids": [48477, 22500, 52383, 128329, 130717, 4787, 72736, 84359, 17731, 128324, 647, 4882], "top_tokens": [" Vietnamese", " Vietnam", " Dong", " \u0111\u1ed3ng", " \u0110\u1ed3ng", " North", " dong", " viet", " Viet", " Vi\u1ec7t", " V", " South"]}], "source_token": 22500, "target_token": 27602, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Da", " N", "ang", " is", " the"], "seconds": 1.178803333081305}
{"run_id": "eval-18/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.189337730407715, "target_logp": -22.924907684326172, "log_odds": -20.735569953918457, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.08112934231758118, -2.108208417892456]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-20.58112907409668, -1.723336935043335, -0.0022136487532407045, -0.6182271242141724]}}, "score_trace": {"source": [], "target": []}, "output": " Vietnamese Dong.  What is the currency used in the country", "output_ids": [48477, 52383, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [], "first_top_ids": [48477, 22500, 52383, 128329, 130717, 4787, 72736, 84359, 17731, 128324], "first_top_logits": [26.5, 23.375, 22.5, 21.875, 21.125, 20.0, 19.875, 19.5, 19.25, 19.125], "source_correct": true, "target_correct": false, "seconds": 0.5098618343472481, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.189337730407715, "target_logp": -22.924907684326172, "log_odds": -20.735569953918457, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.08112934231758118, -2.108208417892456]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-20.58112907409668, -1.723336935043335, -0.0022136487532407045, -0.6182271242141724]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese Dong.  What is the currency used in the country", "output_ids": [48477, 52383, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 22500, 52383, 128329, 130717, 4787, 72736, 84359, 17731, 128324], "first_top_logits": [26.5, 23.375, 22.5, 21.875, 21.125, 20.0, 19.875, 19.5, 19.25, 19.125], "source_correct": true, "target_correct": false, "seconds": 0.4267853335477412, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -53.595726013183594, "target_logp": -17.59028434753418, "log_odds": 36.005441665649414, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-52.375003814697266, -1.2207221984863281]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-15.62500286102295, -1.3431336879730225, -0.0022040139883756638, -0.6199427247047424]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.597234725952148, 101.41195678710938, 128.58316040039062, 75.77538299560547, 65.2911605834961, 134.56015014648438, 189.3873291015625, 108.84941864013672, 52.927764892578125, 188.09417724609375, 663.2110595703125, 692.6657104492188, 1101.68603515625, 1346.5040283203125]], "relative_norm": [[0.0006700365920551121, 0.12766817212104797, 0.11125138401985168, 0.0667480081319809, 0.06147543713450432, 0.12446331977844238, 0.1823924332857132, 0.10972931981086731, 0.0551808662712574, 0.1648503690958023, 0.5444290637969971, 0.7463065385818481, 0.9577494859695435, 1.2200396060943604]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.597234725952148, 101.41195678710938, 128.58316040039062, 75.77538299560547, 65.2911605834961, 134.56015014648438, 189.3873291015625, 108.84941864013672, 52.927764892578125, 188.09417724609375, 663.2110595703125, 692.6657104492188, 1101.68603515625, 1346.5040283203125]], "relative_norm": [[0.0006700365920551121, 0.12766817212104797, 0.11125138401985168, 0.0667480081319809, 0.06147543713450432, 0.12446331977844238, 0.1823924332857132, 0.10972931981086731, 0.0551808662712574, 0.1648503690958023, 0.5444290637969971, 0.7463065385818481, 0.9577494859695435, 1.2200396060943604]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Poland's z\u0142oty.  What is the country that", "output_ids": [27602, 594, 58068, 354, 88, 13, 220, 3555, 374, 279, 3146, 429], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 192.3512420654297, 658.3345947265625, 702.608642578125, 1093.475830078125, 1335.8406982421875]], "actual_norm": [[10.597234725952148, 99.88351440429688, 131.52182006835938, 75.39046478271484, 64.74720001220703, 137.087158203125, 193.6033172607422, 107.1884536743164, 51.027610778808594, 192.3404083251953, 658.3079833984375, 702.5701293945312, 1093.3441162109375, 1335.82470703125]], "relative_norm": [[0.0006700365920551121, 0.12761647999286652, 0.11384477466344833, 0.06660604476928711, 0.060886677354574203, 0.12703536450862885, 0.18691280484199524, 0.10772683471441269, 0.05309063941240311, 0.1676095426082611, 0.542010486125946, 0.7597512006759644, 0.9600631594657898, 1.2143634557724]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27602, 3651, 1471, 31984, 14658, 422, 444, 57719, 112020, 16821], "first_top_logits": [35.25, 22.25, 21.0, 19.75, 19.0, 17.875, 17.5, 17.375, 16.875, 16.875], "source_correct": false, "target_correct": true, "seconds": 0.41406579175964, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -10.88010025024414, "target_logp": -25.70142936706543, "log_odds": -14.821329116821289, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-9.25404167175293, -1.626058578491211]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-23.56654167175293, -1.5987800359725952, -0.0017494149506092072, -0.5343594551086426]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.87972640991211, 188.10275268554688, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.574127197265625, 101.4311752319336, 128.6035614013672, 75.83547973632812, 65.24454498291016, 134.52688598632812, 189.44252014160156, 108.841552734375, 52.88261032104492, 188.0638885498047, 663.2756958007812, 692.7234497070312, 1101.58544921875, 1346.35888671875]], "relative_norm": [[0.0006685755797661841, 0.1276923567056656, 0.1112690344452858, 0.06680094450712204, 0.06143154576420784, 0.12443255633115768, 0.18244558572769165, 0.10972139239311218, 0.05513378605246544, 0.16482383012771606, 0.5444821715354919, 0.7463687658309937, 0.9576619863510132, 1.2199081182479858]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.87972640991211, 188.10275268554688, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.574127197265625, 101.4311752319336, 128.6035614013672, 75.83547973632812, 65.24454498291016, 134.52688598632812, 189.44252014160156, 108.841552734375, 52.88261032104492, 188.0638885498047, 663.2756958007812, 692.7234497070312, 1101.58544921875, 1346.35888671875]], "relative_norm": [[0.0006685755797661841, 0.1276923567056656, 0.1112690344452858, 0.06680094450712204, 0.06143154576420784, 0.12443255633115768, 0.18244558572769165, 0.10972139239311218, 0.05513378605246544, 0.16482383012771606, 0.5444821715354919, 0.7463687658309937, 0.9576619863510132, 1.2199081182479858]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Da", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 14235], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 192.3512420654297, 658.3346557617188, 702.608642578125, 1093.4757080078125, 1335.8406982421875]], "actual_norm": [[10.574127197265625, 99.91732788085938, 131.5339813232422, 75.3414077758789, 64.7527847290039, 137.03257751464844, 193.58447265625, 107.20710754394531, 51.06242752075195, 192.32000732421875, 658.3125, 702.560546875, 1093.33837890625, 1335.90283203125]], "relative_norm": [[0.0006685755797661841, 0.12765967845916748, 0.11385530233383179, 0.0665627047419548, 0.060891930013895035, 0.12698478996753693, 0.18689461052417755, 0.10774558782577515, 0.05312686413526535, 0.16759176552295685, 0.5420141816139221, 0.7597408294677734, 0.9600580930709839, 1.2144345045089722]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 134597, 6633, 12122, 220, 48477, 659, 128324, 198, 13], "first_top_logits": [28.125, 21.875, 21.125, 20.25, 19.25, 19.125, 18.625, 18.375, 18.125, 18.0], "source_correct": false, "target_correct": false, "seconds": 0.4344565002247691, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -10.665973663330078, "target_logp": -51.94292449951172, "log_odds": -41.27695083618164, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-8.625198364257812, -2.0407748222351074]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-49.37519836425781, -1.7900376319885254, -0.0021365690045058727, -0.7755514979362488]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.58515739440918, 101.44721984863281, 128.60348510742188, 75.79740142822266, 65.27684020996094, 134.5218963623047, 189.4027862548828, 108.89777374267578, 52.902557373046875, 188.12124633789062, 663.3524169921875, 692.7141723632812, 1101.5836181640625, 1346.3922119140625]], "relative_norm": [[0.0006692729657515883, 0.12771256268024445, 0.11126896739006042, 0.06676740199327469, 0.061461951583623886, 0.12442793697118759, 0.18240731954574585, 0.10977806895971298, 0.05515458434820175, 0.1648740917444229, 0.5445451140403748, 0.7463587522506714, 0.957660436630249, 1.2199382781982422]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.58515739440918, 101.44721984863281, 128.60348510742188, 75.79740142822266, 65.27684020996094, 134.5218963623047, 189.4027862548828, 108.89777374267578, 52.902557373046875, 188.12124633789062, 663.3524169921875, 692.7141723632812, 1101.5836181640625, 1346.3922119140625]], "relative_norm": [[0.0006692729657515883, 0.12771256268024445, 0.11126896739006042, 0.06676740199327469, 0.061461951583623886, 0.12442793697118759, 0.18240731954574585, 0.10977806895971298, 0.05515458434820175, 0.1648740917444229, 0.5445451140403748, 0.7463587522506714, 0.957660436630249, 1.2199382781982422]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnam Dong.  What is the currency used in the country", "output_ids": [22500, 52383, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 192.3512420654297, 658.3345947265625, 702.608642578125, 1093.475830078125, 1335.8406982421875]], "actual_norm": [[10.58515739440918, 99.8964614868164, 131.49807739257812, 75.40767669677734, 64.7639389038086, 137.0272216796875, 193.5645751953125, 107.19935607910156, 51.015113830566406, 192.29261779785156, 658.422119140625, 702.6470336914062, 1093.602783203125, 1335.79541015625]], "relative_norm": [[0.0006692729657515883, 0.1276330202817917, 0.11382422596216202, 0.06662125140428543, 0.060902420431375504, 0.12697981297969818, 0.18687540292739868, 0.10773779451847076, 0.053077638149261475, 0.1675679087638855, 0.5421044230461121, 0.759834349155426, 0.9602903127670288, 1.21433687210083]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [22500, 48477, 17731, 84359, 94054, 220, 19551, 128324, 106676, 3693], "first_top_logits": [32.25, 23.625, 20.5, 19.375, 19.375, 18.625, 17.875, 17.75, 17.25, 17.25], "source_correct": true, "target_correct": false, "seconds": 0.43353404104709625, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.8611712455749512, "target_logp": -18.292232513427734, "log_odds": -16.431061267852783, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.04089805856347084, -1.8202731609344482]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-15.790898323059082, -1.8849694728851318, -0.002148464322090149, -0.6142175793647766]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.60308837890625, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.40640258789062, 108.9019775390625, 52.87971878051758, 188.10275268554688, 663.291748046875, 692.6727294921875, 1101.6849365234375, 1346.36474609375]], "actual_norm": [[10.552916526794434, 101.45048522949219, 128.61045837402344, 75.87791442871094, 65.17778778076172, 134.5554962158203, 189.37551879882812, 108.9405517578125, 52.91693878173828, 188.04356384277344, 663.1246337890625, 692.6433715820312, 1101.6697998046875, 1346.2935791015625]], "relative_norm": [[0.0006672344752587378, 0.12771667540073395, 0.11127500236034393, 0.0668383240699768, 0.06136868894100189, 0.12445901334285736, 0.1823810487985611, 0.10982119292020798, 0.05516957864165306, 0.16480602324008942, 0.5443581342697144, 0.7462824583053589, 0.9577353596687317, 1.219848871231079]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.60308837890625, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.40640258789062, 108.9019775390625, 52.87971878051758, 188.10275268554688, 663.291748046875, 692.6727294921875, 1101.6849365234375, 1346.36474609375]], "actual_norm": [[10.552916526794434, 101.45048522949219, 128.61045837402344, 75.87791442871094, 65.17778778076172, 134.5554962158203, 189.37551879882812, 108.9405517578125, 52.91693878173828, 188.04356384277344, 663.1246337890625, 692.6433715820312, 1101.6697998046875, 1346.2935791015625]], "relative_norm": [[0.0006672344752587378, 0.12771667540073395, 0.11127500236034393, 0.0668383240699768, 0.06136868894100189, 0.12445901334285736, 0.1823810487985611, 0.10982119292020798, 0.05516957864165306, 0.16480602324008942, 0.5443581342697144, 0.7462824583053589, 0.9577353596687317, 1.219848871231079]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese Dong.  What is the name of the country that", "output_ids": [48477, 52383, 13, 220, 3555, 374, 279, 829, 315, 279, 3146, 429], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90184783935547, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321228027344, 51.04629898071289, 192.35122680664062, 658.3345947265625, 702.608642578125, 1093.4757080078125, 1335.840576171875]], "actual_norm": [[10.552916526794434, 99.857666015625, 131.54396057128906, 75.45318603515625, 64.69916534423828, 137.00015258789062, 193.49868774414062, 107.2196044921875, 51.0920524597168, 192.3743133544922, 658.2269897460938, 702.6064453125, 1093.4517822265625, 1335.616455078125]], "relative_norm": [[0.0006672344752587378, 0.12758345901966095, 0.11386394500732422, 0.06666145473718643, 0.06084150820970535, 0.12695473432540894, 0.18681178987026215, 0.10775814205408096, 0.053157687187194824, 0.1676390916109085, 0.5419437885284424, 0.7597904801368713, 0.9601576924324036, 1.2141741514205933]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 8585, 22500, 220, 30, 84359, 17672, 19461, 2097, 26392], "first_top_logits": [23.875, 19.5, 19.5, 18.75, 18.0, 17.25, 17.25, 17.0, 16.75, 16.625], "source_correct": true, "target_correct": false, "seconds": 0.4401494162157178, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.422377109527588, "target_logp": -16.19989776611328, "log_odds": -12.777520656585693, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-1.3352794647216797, -2.087097644805908]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-13.71027946472168, -1.5349688529968262, -0.002267410745844245, -0.9523816108703613]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.23081970214844, 134.5460968017578, 189.40643310546875, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.546387672424316, 101.42137145996094, 128.65260314941406, 75.8039321899414, 65.20276641845703, 134.55233764648438, 189.3370361328125, 108.94488525390625, 52.9619140625, 188.05589294433594, 663.2742309570312, 692.66015625, 1101.555419921875, 1346.406005859375]], "relative_norm": [[0.0006668216665275395, 0.12768001854419708, 0.11131146550178528, 0.06677315384149551, 0.06139220669865608, 0.12445609271526337, 0.182343989610672, 0.10982555896043777, 0.0552164688706398, 0.16481682658195496, 0.5444809198379517, 0.7463005781173706, 0.9576358795166016, 1.219950795173645]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.23081970214844, 134.5460968017578, 189.40643310546875, 108.90198516845703, 52.879722595214844, 188.10276794433594, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.546387672424316, 101.42137145996094, 128.65260314941406, 75.8039321899414, 65.20276641845703, 134.55233764648438, 189.3370361328125, 108.94488525390625, 52.9619140625, 188.05589294433594, 663.2742309570312, 692.66015625, 1101.555419921875, 1346.406005859375]], "relative_norm": [[0.0006668216665275395, 0.12768001854419708, 0.11131146550178528, 0.06677315384149551, 0.06139220669865608, 0.12445609271526337, 0.182343989610672, 0.10982555896043777, 0.0552164688706398, 0.16481682658195496, 0.5444809198379517, 0.7463005781173706, 0.9576358795166016, 1.219950795173645]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese Dong.\n\nAnother fact: The currency used in the country", "output_ids": [48477, 52383, 382, 14037, 2097, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90185546875, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 192.35125732421875, 658.3345947265625, 702.608642578125, 1093.475830078125, 1335.8406982421875]], "actual_norm": [[10.546387672424316, 99.89518737792969, 131.57762145996094, 75.3753890991211, 64.6564712524414, 137.0599822998047, 193.5853271484375, 107.31085205078125, 51.074981689453125, 192.388916015625, 658.4523315429688, 702.5759887695312, 1093.2640380859375, 1335.9232177734375]], "relative_norm": [[0.0006668216665275395, 0.12763139605522156, 0.11389307677745819, 0.06659272313117981, 0.06080136075615883, 0.12701018154621124, 0.1868954449892044, 0.1078498512506485, 0.053139928728342056, 0.16765181720256805, 0.5421293377876282, 0.7597575187683105, 0.9599928259849548, 1.2144529819488525]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 13, 17672, 17731, 1380, 19461, 892, 264, 128324, 22500], "first_top_logits": [18.375, 17.25, 17.25, 17.125, 17.125, 17.0, 16.875, 16.25, 16.0, 15.8125], "source_correct": true, "target_correct": false, "seconds": 0.4251271248795092, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-18/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-18", "split": "eval", "country": "Vietnam", "currency": "Vietnamese dong", "aliases": ["dong", "VND"], "city": "Da Nang", "target_country": "Poland", "target_currency": "Polish zloty", "target_aliases": ["zloty", "PLN", "z\u0142oty"], "prompt": "Fact: The currency used in the country containing Da Nang is the", "source_continuation": " Vietnamese dong", "target_continuation": " Polish zloty"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9456470012664795, "target_logp": -23.022878646850586, "log_odds": -21.077231645584106, "token_scores": {"source": {"ids": [48477, 72736], "log_probs": [-0.19691351056098938, -1.7487335205078125]}, "target": {"ids": [31984, 1147, 9184, 88], "log_probs": [-20.95667839050293, -1.620929479598999, -0.002487186575308442, -0.44278281927108765]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.9019775390625, 52.879722595214844, 188.10275268554688, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.564362525939941, 101.44377899169922, 128.5926971435547, 75.87361145019531, 65.23687744140625, 134.54446411132812, 189.32424926757812, 108.93573760986328, 52.85751724243164, 188.12680053710938, 663.22119140625, 692.58740234375, 1101.7294921875, 1346.365234375]], "relative_norm": [[0.0006679581711068749, 0.12770822644233704, 0.11125963926315308, 0.06683453172445297, 0.06142432615160942, 0.12444881349802017, 0.18233168125152588, 0.10981634259223938, 0.05510762706398964, 0.1648789644241333, 0.5444374084472656, 0.7462221384048462, 0.9577872157096863, 1.2199138402938843]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 101.42243957519531, 128.6031036376953, 75.78732299804688, 65.2308120727539, 134.5460968017578, 189.4064178466797, 108.9019775390625, 52.879722595214844, 188.10275268554688, 663.2918090820312, 692.6727294921875, 1101.68505859375, 1346.3648681640625]], "actual_norm": [[10.564362525939941, 101.44377899169922, 128.5926971435547, 75.87361145019531, 65.23687744140625, 134.54446411132812, 189.32424926757812, 108.93573760986328, 52.85751724243164, 188.12680053710938, 663.22119140625, 692.58740234375, 1101.7294921875, 1346.365234375]], "relative_norm": [[0.0006679581711068749, 0.12770822644233704, 0.11125963926315308, 0.06683453172445297, 0.06142432615160942, 0.12444881349802017, 0.18233168125152588, 0.10981634259223938, 0.05510762706398964, 0.1648789644241333, 0.5444374084472656, 0.7462221384048462, 0.9577872157096863, 1.2199138402938843]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Vietnamese Dong.  What else do we know about Da N", "output_ids": [48477, 52383, 13, 220, 3555, 770, 653, 582, 1414, 911, 14235, 451], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14235, 451, 524, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[10.557646751403809, 99.90184783935547, 131.49581909179688, 75.36540222167969, 64.6932373046875, 137.00802612304688, 193.59304809570312, 107.18321990966797, 51.046302795410156, 192.3512420654297, 658.3345336914062, 702.608642578125, 1093.4757080078125, 1335.8406982421875]], "actual_norm": [[10.564362525939941, 99.9435806274414, 131.49911499023438, 75.4498519897461, 64.73917388916016, 136.97665405273438, 193.52032470703125, 107.17867279052734, 51.03163146972656, 192.38656616210938, 658.4580078125, 702.6539306640625, 1093.54443359375, 1335.78759765625]], "relative_norm": [[0.0006679581711068749, 0.12769322097301483, 0.11382512748241425, 0.06665851175785065, 0.06087912991642952, 0.12693296372890472, 0.18683268129825592, 0.10771700739860535, 0.05309482291340828, 0.1676497757434845, 0.5421339869499207, 0.759841799736023, 0.960239052772522, 1.214329719543457]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [48477, 19551, 94054, 22500, 647, 17731, 220, 128324, 4882, 17692], "first_top_logits": [21.0, 18.0, 18.0, 18.0, 17.0, 16.875, 15.5625, 15.5, 15.3125, 15.0], "source_correct": true, "target_correct": false, "seconds": 0.44861616706475616, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/readout", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 271.0, "target_rank": 13.0, "source_logit": 4.5, "target_logit": 7.84375, "top_ids": [15973, 11413, 8330, 45418, 7065, 3001, 38352, 7513, 6364, 28453, 49029, 4505], "top_tokens": ["currency", " currency", " Australia", ".currency", "ican", "ica", "uenta", " European", " English", " Currency", "ctica", " Europe"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 284.0, "target_rank": 300.5, "source_logit": 4.4375, "target_logit": 4.34375, "top_ids": [11413, 15973, 3001, 96694, 53661, 28453, 5088, 28130, 42189, 26321, 3693, 45418], "top_tokens": [" currency", "currency", "ica", "onis", "cla", " Currency", "urrency", "ija", "etz", "Currency", " American", ".currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 162.5, "target_rank": 76.0, "source_logit": 18.5, "target_logit": 20.5, "top_ids": [11413, 35602, 32488, 85466, 17672, 7513, 90475, 6424, 8330, 103408, 4505, 6489], "top_tokens": [" currency", " currencies", " euros", " Euros", " euro", " European", "(currency", " UK", " Australia", "\u6b27\u5143", " Europe", " international"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 106.5, "target_rank": 59.0, "source_logit": 15.0, "target_logit": 15.625, "top_ids": [11413, 7214, 65001, 13224, 35602, 7855, 21130, 6938, 47412, 74508, 3220, 32488], "top_tokens": [" currency", " foreign", "\"\u2014", " Britain", " currencies", " British", " parliament", " \u00a3", " foreigners", "\u00bb.", " money", " euros"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 1608.0, "target_rank": 2048.0, "source_logit": 2.0, "target_logit": 1.71875, "top_ids": [21251, 55964, 7748, 28453, 3731, 3776, 12751, 56537, 11413, 39405, 5737, 45265], "top_tokens": ["onia", "\u2011", " Indian", " Currency", " Red", " One", " Roman", " _______,", " currency", " Northwest", " Group", "/rest"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1237.0, "target_rank": 1616.5, "source_logit": 2.078125, "target_logit": 1.7109375, "top_ids": [364, 11413, 7179, 479, 11766, 29991, 2932, 21251, 64, 356, 28453, 18174], "top_tokens": [" '", " currency", " International", " G", " Standard", "\u540d\u79f0", " She", "onia", "a", " C", " Currency", "oret"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 957.5, "target_rank": 1245.0, "source_logit": 14.125, "target_logit": 13.6875, "top_ids": [11413, 35602, 5837, 3146, 32488, 5313, 7576, 13867, 43895, 8330, 7214, 17672], "top_tokens": [" currency", " currencies", " countries", " country", " euros", " national", " prices", " county", " tariffs", " Australia", " foreign", " euro"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 203.5, "target_rank": 45.0, "source_logit": 14.625, "target_logit": 16.5, "top_ids": [11413, 35602, 37198, 7855, 7513, 17672, 7214, 5837, 6489, 94936, 105492, 6364], "top_tokens": [" currency", " currencies", " Commonwealth", " British", " European", " euro", " foreign", " countries", " international", " denomination", "\u4e2d\u534e\u4eba\u6c11", " English"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 9245.5, "target_rank": 6976.0, "source_logit": -0.9765625, "target_logit": -0.4765625, "top_ids": [3693, 4787, 4229, 21251, 15503, 89478, 5434, 2598, 6364, 279, 46427, 271], "top_tokens": [" American", " North", " Ge", "onia", " primarily", "\u2026\n", "\u2026\n\n", " called", " English", " the", "/is", "\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 4359.0, "target_rank": 3349.0, "source_logit": 0.0213623046875, "target_logit": 0.462890625, "top_ids": [1852, 5938, 3693, 4102, 4787, 479, 271, 4882, 3639, 6364, 220, 362], "top_tokens": [" same", " German", " American", "\u00a0", " North", " G", "\n\n", " South", " United", " English", " ", " A"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 3712.5, "target_rank": 1809.5, "source_logit": 8.5, "target_logit": 10.125, "top_ids": [11413, 104142, 103408, 35602, 113883, 32488, 28453, 106333, 115160, 90475, 26321, 85466], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " currencies", "\u672c\u56fd", " euros", " Currency", "\u5916\u6c47", "\u5176\u4ed6\u56fd\u5bb6", "(currency", "Currency", " Euros"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 2176.0, "target_rank": 627.0, "source_logit": 4.28125, "target_logit": 6.5625, "top_ids": [104142, 103408, 116866, 112397, 101237, 106333, 113883, 109442, 101406, 106127, 110065, 99599], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u7f8e\u5143", "\u5916\u6c47", "\u672c\u56fd", "\u8d22\u653f\u90e8", "\u8d27\u5e01", "\u6b27\u76df", "\u6c47\u7387", "\u56fd\u5bb6"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 18.0, "target_rank": 165.0, "source_logit": 17.625, "target_logit": 9.375, "top_ids": [11413, 32488, 85466, 112455, 11192, 31984, 17672, 104142, 103408, 28453, 17692, 112020], "top_tokens": [" currency", " euros", " Euros", "\u82f1\u9551", " dollars", " Polish", " euro", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " Currency", " dollar", "\u6ce2\u5170"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 94.5, "source_logit": 21.25, "target_logit": 10.625, "top_ids": [31984, 11413, 112020, 27602, 17672, 104142, 58068, 32488, 17692, 85466, 28453, 11192], "top_tokens": [" Polish", " currency", "\u6ce2\u5170", " Poland", " euro", "\u4eba\u6c11\u5e01", " z\u0142", " euros", " dollar", " Euros", " Currency", " dollars"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 33.0, "target_rank": 79.0, "source_logit": 27.625, "target_logit": 20.625, "top_ids": [11413, 32488, 103408, 112455, 104142, 11192, 35602, 17672, 85466, 28453, 101406, 80189], "top_tokens": [" currency", " euros", "\u6b27\u5143", "\u82f1\u9551", "\u4eba\u6c11\u5e01", " dollars", " currencies", " euro", " Euros", " Currency", "\u8d27\u5e01", " Dollars"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 10.5, "target_rank": 55.0, "source_logit": 24.25, "target_logit": 16.25, "top_ids": [104142, 112020, 103408, 31984, 32488, 11413, 101406, 17672, 35602, 112455, 27602, 45418], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6ce2\u5170", "\u6b27\u5143", " Polish", " euros", " currency", "\u8d27\u5e01", " euro", " currencies", "\u82f1\u9551", " Poland", ".currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 30.0, "target_rank": 1620.0, "source_logit": 17.75, "target_logit": 8.3125, "top_ids": [279, 31984, 58068, 2598, 1863, 19461, 472, 17672, 3881, 112020, 1147, 78252], "top_tokens": [" the", " Polish", " z\u0142", " called", " Z", " Euro", " H", " euro", " known", "\u6ce2\u5170", " z", " Pound"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 16.0, "target_rank": 1209.5, "source_logit": 16.625, "target_logit": 7.28125, "top_ids": [58068, 31984, 1863, 19461, 17672, 1147, 112020, 44029, 10315, 105828, 7513, 220], "top_tokens": [" z\u0142", " Polish", " Z", " Euro", " euro", " z", "\u6ce2\u5170", " polish", " PL", "\u5179", " European", " "]}], "source_token": 27602, "target_token": 42776, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Krak", "ow", " is", " the"], "seconds": 1.379806624725461}
{"run_id": "eval-19/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.518134832382202, "target_logp": -10.302406311035156, "log_odds": -7.784271478652954, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-1.0733610391616821, -1.433370590209961, -0.011089124716818333, -0.0003143055073451251]}, "target": {"ids": [56769, 369, 396], "log_probs": [-10.07336139678955, -0.22868786752223969, -0.00035672978265210986]}}, "score_trace": {"source": [], "target": []}, "output": " Polish Zloty.  What is the answer to the", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 374, 279, 4226, 311, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [], "first_top_ids": [58068, 31984, 1863, 19461, 17672, 1147, 112020, 44029, 10315, 105828], "first_top_logits": [25.25, 25.25, 25.0, 22.875, 21.625, 21.5, 21.0, 19.375, 19.25, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.469761457759887, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.518134832382202, "target_logp": -10.302406311035156, "log_odds": -7.784271478652954, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-1.0733610391616821, -1.433370590209961, -0.011089124716818333, -0.0003143055073451251]}, "target": {"ids": [56769, 369, 396], "log_probs": [-10.07336139678955, -0.22868786752223969, -0.00035672978265210986]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What is the answer to the", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 374, 279, 4226, 311, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58068, 31984, 1863, 19461, 17672, 1147, 112020, 44029, 10315, 105828], "first_top_logits": [25.25, 25.25, 25.0, 22.875, 21.625, 21.5, 21.0, 19.375, 19.25, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.4231410827487707, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -20.05454444885254, "target_logp": -0.22476914525032043, "log_odds": 19.82977530360222, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-18.76432228088379, -1.2788302898406982, -0.010985254310071468, -0.00040892345714382827]}, "target": {"ids": [56769, 369, 396], "log_probs": [-0.014322223141789436, -0.21007733047008514, -0.0003695997002068907]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.960723876953125, 145.37890625, 48.967529296875, 164.45750427246094, 6.395989894866943, 180.43714904785156, 1.6620593070983887, 71.4063720703125, 105.8711166381836, 131.602294921875, 406.1712341308594, 371.4158630371094, 489.2021484375]], "relative_norm": [[0.002084024017676711, 0.18301844596862793, 0.042367175221443176, 0.14486512541770935, 0.006022197660058737, 0.16689790785312653, 0.001600672141648829, 0.07198359817266464, 0.11037798225879669, 0.11351040750741959, 0.44430941343307495, 0.3307028114795685, 0.4491851329803467]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.960723876953125, 145.37890625, 48.967529296875, 164.45750427246094, 6.395989894866943, 180.43714904785156, 1.6620593070983887, 71.4063720703125, 105.8711166381836, 131.602294921875, 406.1712341308594, 371.4158630371094, 489.2021484375]], "relative_norm": [[0.002084024017676711, 0.18301844596862793, 0.042367175221443176, 0.14486512541770935, 0.006022197660058737, 0.16689790785312653, 0.001600672141648829, 0.07198359817266464, 0.11037798225879669, 0.11351040750741959, 0.44430941343307495, 0.3307028114795685, 0.4491851329803467]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.\n\nFact: The currency used in the country", "output_ids": [56769, 369, 396, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 132.65907287597656, 409.2779235839844, 378.5971374511719, 486.90625]], "actual_norm": [[32.960723876953125, 142.39425659179688, 47.1733283996582, 163.4515838623047, 6.046560764312744, 182.67431640625, 4.492311000823975, 66.50703430175781, 105.50971984863281, 132.71218872070312, 409.27142333984375, 378.6070251464844, 486.945556640625]], "relative_norm": [[0.002084024017676711, 0.18193046748638153, 0.04083304852247238, 0.14440637826919556, 0.005686037242412567, 0.16927987337112427, 0.004337066784501076, 0.06684108823537827, 0.10977544635534286, 0.1145387515425682, 0.4482775926589966, 0.3370567560195923, 0.4470009505748749]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 42776, 89375, 103670, 472, 19461, 17672, 58068, 33150, 99070], "first_top_logits": [31.875, 27.625, 22.5, 21.375, 21.0, 20.5, 20.375, 17.125, 17.0, 16.5], "source_correct": false, "target_correct": true, "seconds": 0.420440583024174, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.1375997066497803, "target_logp": -11.89919376373291, "log_odds": -8.76159405708313, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-1.5260188579559326, -1.6024608612060547, -0.008718283846974373, -0.0004017737810499966]}, "target": {"ids": [56769, 369, 396], "log_probs": [-11.651019096374512, -0.24784909188747406, -0.0003250309091527015]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.985652923583984, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84649658203125, 131.567138671875, 406.2442321777344, 371.46832275390625, 489.17327880859375]], "actual_norm": [[32.94980239868164, 145.3740692138672, 49.03316879272461, 164.48281860351562, 6.458939075469971, 180.4295196533203, 1.635107398033142, 71.37244415283203, 105.81529998779297, 131.54989624023438, 406.1874694824219, 371.45684814453125, 489.26165771484375]], "relative_norm": [[0.0020833334419876337, 0.18301235139369965, 0.04242396727204323, 0.14488743245601654, 0.0060814679600298405, 0.16689084470272064, 0.0015747157158330083, 0.07194939255714417, 0.11031978577375412, 0.11346521228551865, 0.444327175617218, 0.330739289522171, 0.4492397904396057]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.985652923583984, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84649658203125, 131.567138671875, 406.2442321777344, 371.46832275390625, 489.17327880859375]], "actual_norm": [[32.94980239868164, 145.3740692138672, 49.03316879272461, 164.48281860351562, 6.458939075469971, 180.4295196533203, 1.635107398033142, 71.37244415283203, 105.81529998779297, 131.54989624023438, 406.1874694824219, 371.45684814453125, 489.26165771484375]], "relative_norm": [[0.0020833334419876337, 0.18301235139369965, 0.04242396727204323, 0.14488743245601654, 0.0060814679600298405, 0.16689084470272064, 0.0015747157158330083, 0.07194939255714417, 0.11031978577375412, 0.11346521228551865, 0.444327175617218, 0.330739289522171, 0.4492397904396057]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency used in the country containing", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 132.65907287597656, 409.2779235839844, 378.5971374511719, 486.90625]], "actual_norm": [[32.94980239868164, 142.4273681640625, 47.152523040771484, 163.43370056152344, 6.126401901245117, 182.63926696777344, 4.492606163024902, 66.5239028930664, 105.5048599243164, 132.6384735107422, 409.2934875488281, 378.6566162109375, 486.9097595214844]], "relative_norm": [[0.0020833334419876337, 0.18197277188301086, 0.04081504046916962, 0.14439056813716888, 0.0057611181400716305, 0.16924740374088287, 0.004337351303547621, 0.06685803830623627, 0.10977038741111755, 0.11447513103485107, 0.448301762342453, 0.33710089325904846, 0.44696810841560364]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 31984, 7513, 17672, 19461, 112020, 58068, 44029, 1863, 6633], "first_top_logits": [26.625, 25.5, 23.5, 23.5, 23.375, 23.25, 22.125, 21.0, 20.875, 20.5], "source_correct": false, "target_correct": false, "seconds": 0.47643808322027326, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1995043754577637, "target_logp": -30.565982818603516, "log_odds": -28.366478443145752, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.4962894022464752, -1.6918790340423584, -0.010997634381055832, -0.0003383779258001596]}, "target": {"ids": [56769, 369, 396], "log_probs": [-30.30879020690918, -0.2568444013595581, -0.00034850722295232117]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.9312744140625, 145.43594360351562, 48.9804573059082, 164.47061157226562, 6.410210132598877, 180.39752197265625, 1.662532925605774, 71.40785217285156, 105.845458984375, 131.5349884033203, 406.2857666015625, 371.49163818359375, 489.18841552734375]], "relative_norm": [[0.002082162071019411, 0.1830902397632599, 0.04237836226820946, 0.14487667381763458, 0.006035586819052696, 0.16686125099658966, 0.0016011283732950687, 0.07198508828878403, 0.11035122722387314, 0.11345235258340836, 0.4444347023963928, 0.3307702839374542, 0.4491725265979767]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.9312744140625, 145.43594360351562, 48.9804573059082, 164.47061157226562, 6.410210132598877, 180.39752197265625, 1.662532925605774, 71.40785217285156, 105.845458984375, 131.5349884033203, 406.2857666015625, 371.49163818359375, 489.18841552734375]], "relative_norm": [[0.002082162071019411, 0.1830902397632599, 0.04237836226820946, 0.14487667381763458, 0.006035586819052696, 0.16686125099658966, 0.0016011283732950687, 0.07198508828878403, 0.11035122722387314, 0.11345235258340836, 0.4444347023963928, 0.3307702839374542, 0.4491725265979767]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What is the answer to the", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 374, 279, 4226, 311, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 132.65907287597656, 409.2779235839844, 378.5971374511719, 486.90625]], "actual_norm": [[32.9312744140625, 142.42698669433594, 47.13922882080078, 163.4822235107422, 6.057154178619385, 182.66525268554688, 4.481956958770752, 66.53463745117188, 105.5003433227539, 132.7130584716797, 409.2364807128906, 378.5788879394531, 486.9484558105469]], "relative_norm": [[0.002082162071019411, 0.18197228014469147, 0.04080353304743767, 0.14443343877792358, 0.005695999134331942, 0.16927148401737213, 0.0043270704336464405, 0.06686882674694061, 0.1097656860947609, 0.1145395040512085, 0.4482393264770508, 0.3370317220687866, 0.44700363278388977]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 27602, 393, 1863, 19461, 58068, 44029, 17672, 10315, 1471], "first_top_logits": [25.25, 24.375, 22.75, 22.25, 21.875, 20.75, 20.5, 20.5, 20.375, 20.25], "source_correct": true, "target_correct": false, "seconds": 0.4620416248217225, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.8169820308685303, "target_logp": -10.327839851379395, "log_odds": -7.510857820510864, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-1.417628526687622, -1.3879778385162354, -0.01101979985833168, -0.00035589560866355896]}, "target": {"ids": [56769, 369, 396], "log_probs": [-10.105128288269043, -0.22231750190258026, -0.00039402826223522425]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.392578125, 48.985652923583984, 164.44931030273438, 6.291029930114746, 180.39573669433594, 1.6272830963134766, 71.39640808105469, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.4682922363281, 489.1732177734375]], "actual_norm": [[32.919979095458984, 145.3661651611328, 48.982269287109375, 164.5002899169922, 6.37715482711792, 180.38011169433594, 1.6790040731430054, 71.34098815917969, 105.8484115600586, 131.59100341796875, 406.2429504394531, 371.4906921386719, 489.21685791015625]], "relative_norm": [[0.002081447746604681, 0.18300239741802216, 0.04237993061542511, 0.14490282535552979, 0.006004463415592909, 0.1668451428413391, 0.0016169911250472069, 0.07191768288612366, 0.11035430431365967, 0.1135006695985794, 0.44438785314559937, 0.330769419670105, 0.4491986334323883]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.392578125, 48.985652923583984, 164.44931030273438, 6.291029930114746, 180.39573669433594, 1.6272830963134766, 71.39640808105469, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.4682922363281, 489.1732177734375]], "actual_norm": [[32.919979095458984, 145.3661651611328, 48.982269287109375, 164.5002899169922, 6.37715482711792, 180.38011169433594, 1.6790040731430054, 71.34098815917969, 105.8484115600586, 131.59100341796875, 406.2429504394531, 371.4906921386719, 489.21685791015625]], "relative_norm": [[0.002081447746604681, 0.18300239741802216, 0.04237993061542511, 0.14490282535552979, 0.006004463415592909, 0.1668451428413391, 0.0016169911250472069, 0.07191768288612366, 0.11035430431365967, 0.1135006695985794, 0.44438785314559937, 0.330769419670105, 0.4491986334323883]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Zloty.  What is the name of the country", "output_ids": [1863, 9184, 88, 13, 220, 3555, 374, 279, 829, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131103515625, 163.45777893066406, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.50963592529297, 132.65907287597656, 409.2779235839844, 378.5971374511719, 486.9062194824219]], "actual_norm": [[32.919979095458984, 142.45469665527344, 47.12263107299805, 163.44210815429688, 6.064997673034668, 182.65150451660156, 4.511297225952148, 66.51761627197266, 105.5612564086914, 132.68190002441406, 409.372802734375, 378.68536376953125, 486.85699462890625]], "relative_norm": [[0.002081447746604681, 0.18200768530368805, 0.0407891683280468, 0.14439800381660461, 0.005703375209122896, 0.1692587435245514, 0.00435539660975337, 0.06685172021389008, 0.10982906818389893, 0.1145126149058342, 0.44838863611221313, 0.3371264934539795, 0.44691964983940125]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1863, 31984, 58068, 19461, 17672, 1147, 112020, 472, 105828, 220], "first_top_logits": [24.625, 23.75, 22.875, 22.5, 21.25, 20.625, 19.125, 19.0, 17.375, 17.125], "source_correct": true, "target_correct": false, "seconds": 0.46717349998652935, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1758532524108887, "target_logp": -11.325376510620117, "log_odds": -9.149523258209229, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.5589622855186462, -1.6025357246398926, -0.013914601877331734, -0.0004406196530908346]}, "target": {"ids": [56769, 369, 396], "log_probs": [-11.121461868286133, -0.2035331279039383, -0.0003812778159044683]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.97646713256836, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.395751953125, 1.6272832155227661, 71.39642333984375, 105.84649658203125, 131.56715393066406, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.968055725097656, 145.35513305664062, 48.96630859375, 164.46690368652344, 6.37249231338501, 180.36148071289062, 1.675033450126648, 71.3723373413086, 105.8109130859375, 131.56414794921875, 406.2508544921875, 371.4989013671875, 489.08819580078125]], "relative_norm": [[0.002084487583488226, 0.1829885095357895, 0.04236612096428871, 0.14487341046333313, 0.0060000731609761715, 0.16682791709899902, 0.0016131671145558357, 0.07194928824901581, 0.11031521111726761, 0.11347750574350357, 0.4443964958190918, 0.33077675104141235, 0.4490804970264435]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.97646713256836, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.395751953125, 1.6272832155227661, 71.39642333984375, 105.84649658203125, 131.56715393066406, 406.2442626953125, 371.46832275390625, 489.1732482910156]], "actual_norm": [[32.968055725097656, 145.35513305664062, 48.96630859375, 164.46690368652344, 6.37249231338501, 180.36148071289062, 1.675033450126648, 71.3723373413086, 105.8109130859375, 131.56414794921875, 406.2508544921875, 371.4989013671875, 489.08819580078125]], "relative_norm": [[0.002084487583488226, 0.1829885095357895, 0.04236612096428871, 0.14487341046333313, 0.0060000731609761715, 0.16682791709899902, 0.0016131671145558357, 0.07194928824901581, 0.11031521111726761, 0.11347750574350357, 0.4443964958190918, 0.33077675104141235, 0.4490804970264435]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What is the answer to the", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 374, 279, 4226, 311, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.97646713256836, 142.39881896972656, 47.131107330322266, 163.4578094482422, 5.96571159362793, 182.67059326171875, 4.409332275390625, 66.5072021484375, 105.5096435546875, 132.65908813476562, 409.2779541015625, 378.59716796875, 486.9062805175781]], "actual_norm": [[32.968055725097656, 142.43861389160156, 47.145652770996094, 163.4409942626953, 6.10033655166626, 182.65121459960938, 4.498037815093994, 66.49507904052734, 105.4824447631836, 132.67706298828125, 409.29010009765625, 378.65057373046875, 486.8729248046875]], "relative_norm": [[0.002084487583488226, 0.18198713660240173, 0.04080909490585327, 0.14439702033996582, 0.005736606661230326, 0.1692584753036499, 0.0043425955809652805, 0.06682907044887543, 0.10974706709384918, 0.11450843513011932, 0.44829803705215454, 0.3370955288410187, 0.4469342827796936]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 17672, 19461, 58068, 1863, 393, 7513, 112020, 37088, 1147], "first_top_logits": [23.5, 22.75, 22.375, 21.375, 20.0, 18.625, 18.125, 18.125, 17.0, 17.0], "source_correct": true, "target_correct": false, "seconds": 0.48387029161676764, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-19/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-19", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Krakow", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Krakow is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1378214359283447, "target_logp": -9.822365760803223, "log_odds": -7.684544324874878, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.8885445594787598, -1.236651062965393, -0.012379003688693047, -0.0002470903273206204]}, "target": {"ids": [56769, 369, 396], "log_probs": [-9.638544082641602, -0.18342863023281097, -0.0003927174839191139]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44931030273438, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.4682922363281, 489.1732177734375]], "actual_norm": [[32.92869567871094, 145.38853454589844, 48.972450256347656, 164.42532348632812, 6.3936262130737305, 180.3600311279297, 1.6585209369659424, 71.43326568603516, 105.8650131225586, 131.55960083007812, 406.2473449707031, 371.52105712890625, 489.1662292480469]], "relative_norm": [[0.0020819988567382097, 0.18303056061267853, 0.04237143322825432, 0.14483678340911865, 0.006019971799105406, 0.16682657599449158, 0.0015972645487636328, 0.07201071083545685, 0.11037161946296692, 0.11347358673810959, 0.44439268112182617, 0.330796480178833, 0.4491521418094635]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44931030273438, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 131.567138671875, 406.2442626953125, 371.4682922363281, 489.1732177734375]], "actual_norm": [[32.92869567871094, 145.38853454589844, 48.972450256347656, 164.42532348632812, 6.3936262130737305, 180.3600311279297, 1.6585209369659424, 71.43326568603516, 105.8650131225586, 131.55960083007812, 406.2473449707031, 371.52105712890625, 489.1662292480469]], "relative_norm": [[0.0020819988567382097, 0.18303056061267853, 0.04237143322825432, 0.14483678340911865, 0.006019971799105406, 0.16682657599449158, 0.0015972645487636328, 0.07201071083545685, 0.11037161946296692, 0.11347358673810959, 0.44439268112182617, 0.330796480178833, 0.4491521418094635]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What would be the best way", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 1035, 387, 279, 1850, 1616], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 88443, 363, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.9657111167907715, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.50963592529297, 132.65907287597656, 409.2779235839844, 378.5971374511719, 486.90625]], "actual_norm": [[32.92869567871094, 142.43060302734375, 47.1179313659668, 163.47079467773438, 6.083796977996826, 182.6687774658203, 4.489026069641113, 66.54058837890625, 105.51892852783203, 132.70040893554688, 409.1861877441406, 378.636474609375, 486.94622802734375]], "relative_norm": [[0.0020819988567382097, 0.18197689950466156, 0.04078510031104088, 0.14442335069179535, 0.005721053574234247, 0.16927474737167358, 0.004333895165473223, 0.06687480956315994, 0.10978502780199051, 0.114528588950634, 0.44818422198295593, 0.33708298206329346, 0.447001576423645]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 1863, 19461, 58068, 17672, 112020, 1147, 105828, 10315, 422], "first_top_logits": [23.5, 23.375, 21.5, 21.125, 19.625, 19.375, 19.125, 17.5, 17.375, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.4739802498370409, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/readout", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 319.5, "target_rank": 14.0, "source_logit": 4.21875, "target_logit": 7.5, "top_ids": [11413, 45418, 15973, 8330, 38352, 7513, 7065, 6364, 28453, 4505, 3001, 56769], "top_tokens": [" currency", ".currency", "currency", " Australia", "uenta", " European", "ican", " English", " Currency", " Europe", "ica", " Hungarian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 202.0, "target_rank": 157.5, "source_logit": 5.0625, "target_logit": 5.4375, "top_ids": [11413, 15973, 28453, 3001, 3693, 15154, 45418, 5088, 96694, 6364, 7065, 26321], "top_tokens": [" currency", "currency", " Currency", "ica", " American", " Spanish", ".currency", "urrency", "onis", " English", "ican", "Currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 251.0, "target_rank": 166.0, "source_logit": 17.0, "target_logit": 17.875, "top_ids": [11413, 35602, 32488, 7513, 12463, 91984, 29973, 3263, 90475, 7214, 26138, 85466], "top_tokens": [" currency", " currencies", " euros", " European", " colour", " licences", " licence", "\".", "(currency", " foreign", " colours", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 65.5, "target_rank": 51.5, "source_logit": 17.25, "target_logit": 17.625, "top_ids": [11413, 35602, 7214, 13224, 47412, 7855, 32488, 3220, 17672, 9625, 21130, 37198], "top_tokens": [" currency", " currencies", " foreign", " Britain", " foreigners", " British", " euros", " money", " euro", " France", " parliament", " Commonwealth"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 2130.0, "target_rank": 4041.0, "source_logit": 1.7734375, "target_logit": 0.89453125, "top_ids": [21251, 3776, 3731, 603, 67476, 7748, 76231, 5737, 28453, 55964, 46804, 21525], "top_tokens": ["onia", " One", " Red", "one", "'),('", " Indian", " Maritime", " Group", " Currency", "\u2011", " predominantly", " Ring"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1353.5, "target_rank": 2143.0, "source_logit": 2.078125, "target_logit": 1.4375, "top_ids": [7179, 29991, 364, 2932, 479, 21251, 18174, 11413, 11766, 64, 6489, 1269], "top_tokens": [" International", "\u540d\u79f0", " '", " She", " G", "onia", "oret", " currency", " Standard", "a", " international", "_name"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1384.5, "target_rank": 2868.0, "source_logit": 13.125, "target_logit": 12.0625, "top_ids": [11413, 35602, 5313, 13867, 3146, 3283, 5837, 29970, 30176, 6290, 32488, 7214], "top_tokens": [" currency", " currencies", " national", " county", " country", " city", " countries", " tourist", " counties", " town", " euros", " foreign"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 130.5, "target_rank": 30.0, "source_logit": 15.4375, "target_logit": 17.125, "top_ids": [37198, 11413, 17672, 35602, 7513, 7855, 5837, 6364, 6489, 94936, 5313, 32488], "top_tokens": [" Commonwealth", " currency", " euro", " currencies", " European", " British", " countries", " English", " international", " denomination", " national", " euros"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 9519.0, "target_rank": 11802.5, "source_logit": -0.703125, "target_logit": -1.1015625, "top_ids": [2598, 15503, 3693, 4229, 46804, 4787, 5023, 89478, 279, 111012, 5434, 12022], "top_tokens": [" called", " primarily", " American", " Ge", " predominantly", " North", " currently", "\u2026\n", " the", "\u4e16\u754c\u4e0a\u6700", "\u2026\n\n", " \u2026\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 6935.5, "target_rank": 6051.5, "source_logit": -0.5703125, "target_logit": -0.33203125, "top_ids": [1852, 5938, 3693, 4102, 271, 4787, 479, 3639, 4882, 5055, 18174, 7179], "top_tokens": [" same", " German", " American", "\u00a0", "\n\n", " North", " G", " United", " South", " National", "oret", " International"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 5222.0, "target_rank": 4883.0, "source_logit": 7.375, "target_logit": 7.53125, "top_ids": [11413, 103408, 104142, 35602, 28453, 32488, 90475, 26321, 113883, 101406, 101237, 106333], "top_tokens": [" currency", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " currencies", " Currency", " euros", "(currency", "Currency", "\u672c\u56fd", "\u8d27\u5e01", "\u7f8e\u5143", "\u5916\u6c47"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 2567.0, "target_rank": 922.0, "source_logit": 3.375, "target_logit": 5.25, "top_ids": [104142, 103408, 116866, 112397, 106127, 109442, 106333, 101237, 99599, 110065, 101406, 113883], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u76f4\u8f96\u5e02", "\u56fd\u5bb6\u6807\u51c6", "\u6b27\u76df", "\u8d22\u653f\u90e8", "\u5916\u6c47", "\u7f8e\u5143", "\u56fd\u5bb6", "\u6c47\u7387", "\u8d27\u5e01", "\u672c\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 33.5, "target_rank": 798.5, "source_logit": 16.25, "target_logit": 6.25, "top_ids": [11413, 2598, 112455, 32488, 11192, 85466, 104142, 103408, 28453, 17672, 17692, 31984], "top_tokens": [" currency", " called", "\u82f1\u9551", " euros", " dollars", " Euros", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " Currency", " euro", " dollar", " Polish"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 415.0, "source_logit": 20.25, "target_logit": 6.8125, "top_ids": [31984, 11413, 112020, 104142, 27602, 17672, 17692, 28453, 43680, 58068, 19551, 103408], "top_tokens": [" Polish", " currency", "\u6ce2\u5170", "\u4eba\u6c11\u5e01", " Poland", " euro", " dollar", " Currency", " Danish", " z\u0142", " USD", "\u6b27\u5143"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 39.0, "target_rank": 269.0, "source_logit": 24.125, "target_logit": 15.3125, "top_ids": [11413, 32488, 112455, 103408, 104142, 35602, 11192, 28453, 101406, 17672, 85466, 17692], "top_tokens": [" currency", " euros", "\u82f1\u9551", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " currencies", " dollars", " Currency", "\u8d27\u5e01", " euro", " Euros", " dollar"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 13.0, "target_rank": 149.5, "source_logit": 23.5, "target_logit": 12.5, "top_ids": [104142, 103408, 112020, 31984, 11413, 101406, 35602, 32488, 17672, 28453, 112455, 45418], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u6ce2\u5170", " Polish", " currency", "\u8d27\u5e01", " currencies", " euros", " euro", " Currency", "\u82f1\u9551", ".currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 49.5, "target_rank": 4365.5, "source_logit": 16.875, "target_logit": 6.28125, "top_ids": [279, 2598, 31984, 3881, 1863, 58068, 472, 19461, 112020, 1147, 305, 1083], "top_tokens": [" the", " called", " Polish", " known", " Z", " z\u0142", " H", " Euro", "\u6ce2\u5170", " z", " h", " also"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 33.5, "target_rank": 4380.5, "source_logit": 15.375, "target_logit": 4.78125, "top_ids": [31984, 1863, 58068, 19461, 1147, 112020, 17672, 10315, 7513, 43680, 105828, 44029], "top_tokens": [" Polish", " Z", " z\u0142", " Euro", " z", "\u6ce2\u5170", " euro", " PL", " European", " Danish", "\u5179", " polish"]}], "source_token": 27602, "target_token": 42776, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " G", "d", "ansk", " is", " the"], "seconds": 1.1644671251997352}
{"run_id": "eval-20/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1272010803222656, "target_logp": -12.313374519348145, "log_odds": -11.186173439025879, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.10483019798994064, -1.0091016292572021, -0.013117918744683266, -0.00015138433082029223]}, "target": {"ids": [56769, 369, 396], "log_probs": [-12.042329788208008, -0.27067244052886963, -0.00037174468161538243]}}, "score_trace": {"source": [], "target": []}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [], "first_top_ids": [31984, 1863, 58068, 19461, 1147, 112020, 17672, 10315, 7513, 43680], "first_top_logits": [27.0, 24.125, 23.5, 22.125, 20.875, 20.625, 20.375, 20.375, 19.875, 19.5], "source_correct": true, "target_correct": false, "seconds": 0.4776372089982033, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1272010803222656, "target_logp": -12.313374519348145, "log_odds": -11.186173439025879, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.10483019798994064, -1.0091016292572021, -0.013117918744683266, -0.00015138433082029223]}, "target": {"ids": [56769, 369, 396], "log_probs": [-12.042329788208008, -0.27067244052886963, -0.00037174468161538243]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 1863, 58068, 19461, 1147, 112020, 17672, 10315, 7513, 43680], "first_top_logits": [27.0, 24.125, 23.5, 22.125, 20.875, 20.625, 20.375, 20.375, 19.875, 19.5], "source_correct": true, "target_correct": false, "seconds": 0.44807229097932577, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -26.547971725463867, "target_logp": -0.27337875962257385, "log_odds": 26.274592965841293, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-25.690229415893555, -0.8445643186569214, -0.01295448187738657, -0.00022277733660303056]}, "target": {"ids": [56769, 369, 396], "log_probs": [-0.033978600054979324, -0.23902542889118195, -0.0003747238079085946]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.960723876953125, 145.37890625, 48.967529296875, 164.45750427246094, 6.395989894866943, 180.43714904785156, 1.6620593070983887, 71.4063720703125, 105.8711166381836, 32.014244079589844, 17.51239585876465, 549.3076782226562, 492.9049072265625, 636.0220336914062]], "relative_norm": [[0.002084024017676711, 0.18301844596862793, 0.042367175221443176, 0.14486512541770935, 0.006022197660058737, 0.16689790785312653, 0.001600672141648829, 0.07198359817266464, 0.11037798225879669, 0.030578773468732834, 0.014368874952197075, 0.591857373714447, 0.438550740480423, 0.6182275414466858]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.960723876953125, 145.37890625, 48.967529296875, 164.45750427246094, 6.395989894866943, 180.43714904785156, 1.6620593070983887, 71.4063720703125, 105.8711166381836, 32.014244079589844, 17.51239585876465, 549.3076782226562, 492.9049072265625, 636.0220336914062]], "relative_norm": [[0.002084024017676711, 0.18301844596862793, 0.042367175221443176, 0.14486512541770935, 0.006022197660058737, 0.16689790785312653, 0.001600672141648829, 0.07198359817266464, 0.11037798225879669, 0.030578773468732834, 0.014368874952197075, 0.591857373714447, 0.438550740480423, 0.6182275414466858]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.\n\nFact: The currency used in the country", "output_ids": [56769, 369, 396, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 31.316137313842773, 8.941094398498535, 539.4290161132812, 490.3675842285156, 632.1050415039062]], "actual_norm": [[32.960723876953125, 142.39425659179688, 47.1733283996582, 163.4515838623047, 6.046560764312744, 182.67431640625, 4.492311000823975, 66.50703430175781, 105.50971984863281, 31.36285972595215, 9.110335350036621, 539.3775024414062, 490.412353515625, 632.0524291992188]], "relative_norm": [[0.002084024017676711, 0.18193046748638153, 0.04083304852247238, 0.14440637826919556, 0.005686037242412567, 0.16927987337112427, 0.004337066784501076, 0.06684108823537827, 0.10977544635534286, 0.03003772348165512, 0.007469880394637585, 0.5817826986312866, 0.4376515746116638, 0.6156957745552063]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 42776, 89375, 103670, 17672, 19461, 18295, 472, 25523, 1752], "first_top_logits": [30.75, 27.375, 22.125, 21.625, 18.875, 18.625, 17.75, 17.625, 17.25, 16.5], "source_correct": false, "target_correct": true, "seconds": 0.4225407079793513, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.3290789127349854, "target_logp": -15.068867683410645, "log_odds": -11.73978877067566, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-2.03364634513855, -1.2875518798828125, -0.007667633704841137, -0.0002131234941771254]}, "target": {"ids": [56769, 369, 396], "log_probs": [-14.783646583557129, -0.2847996652126312, -0.0004213160718791187]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.985652923583984, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84649658203125, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.94980239868164, 145.3740692138672, 49.03316879272461, 164.48281860351562, 6.458939075469971, 180.4295196533203, 1.635107398033142, 71.37244415283203, 105.81529998779297, 32.03505325317383, 17.468870162963867, 549.3258056640625, 492.8375549316406, 636.0562744140625]], "relative_norm": [[0.0020833334419876337, 0.18301235139369965, 0.04242396727204323, 0.14488743245601654, 0.0060814679600298405, 0.16689084470272064, 0.0015747157158330083, 0.07194939255714417, 0.11031978577375412, 0.030598649755120277, 0.01433316245675087, 0.5918768644332886, 0.4384908080101013, 0.6182608008384705]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.985652923583984, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84649658203125, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.94980239868164, 145.3740692138672, 49.03316879272461, 164.48281860351562, 6.458939075469971, 180.4295196533203, 1.635107398033142, 71.37244415283203, 105.81529998779297, 32.03505325317383, 17.468870162963867, 549.3258056640625, 492.8375549316406, 636.0562744140625]], "relative_norm": [[0.0020833334419876337, 0.18301235139369965, 0.04242396727204323, 0.14488743245601654, 0.0060814679600298405, 0.16689084470272064, 0.0015747157158330083, 0.07194939255714417, 0.11031978577375412, 0.030598649755120277, 0.01433316245675087, 0.5918768644332886, 0.4384908080101013, 0.6182608008384705]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table of contents.  What is the currency of the country", "output_ids": [1965, 315, 8794, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 31.31613540649414, 8.941094398498535, 539.4290161132812, 490.3675537109375, 632.1050415039062]], "actual_norm": [[32.94980239868164, 142.4273681640625, 47.152523040771484, 163.43370056152344, 6.126401901245117, 182.63926696777344, 4.492606163024902, 66.5239028930664, 105.5048599243164, 31.33983612060547, 9.158201217651367, 539.4419555664062, 490.3182678222656, 632.0358276367188]], "relative_norm": [[0.0020833334419876337, 0.18197277188301086, 0.04081504046916962, 0.14439056813716888, 0.0057611181400716305, 0.16924740374088287, 0.004337351303547621, 0.06685803830623627, 0.10977038741111755, 0.030015673488378525, 0.007509127259254456, 0.5818521976470947, 0.4375675916671753, 0.6156795620918274]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 31984, 7513, 19461, 17672, 112020, 6633, 1140, 220, 58068], "first_top_logits": [28.875, 27.125, 24.25, 23.375, 22.75, 22.625, 22.5, 21.875, 20.5, 20.125], "source_correct": false, "target_correct": false, "seconds": 0.4261324158869684, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9518566131591797, "target_logp": -36.266231536865234, "log_odds": -34.314374923706055, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.6301040053367615, -1.3048423528671265, -0.016654858365654945, -0.0002554328821133822]}, "target": {"ids": [56769, 369, 396], "log_probs": [-35.880104064941406, -0.38568565249443054, -0.0004435985756572336]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.9312744140625, 145.43594360351562, 48.9804573059082, 164.47061157226562, 6.410210132598877, 180.39752197265625, 1.662532925605774, 71.40785217285156, 105.845458984375, 32.0539665222168, 17.519691467285156, 549.3359375, 492.8629150390625, 636.0932006835938]], "relative_norm": [[0.002082162071019411, 0.1830902397632599, 0.04237836226820946, 0.14487667381763458, 0.006035586819052696, 0.16686125099658966, 0.0016011283732950687, 0.07198508828878403, 0.11035122722387314, 0.03061671555042267, 0.014374860562384129, 0.5918878316879272, 0.4385133683681488, 0.61829674243927]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.9312744140625, 145.43594360351562, 48.9804573059082, 164.47061157226562, 6.410210132598877, 180.39752197265625, 1.662532925605774, 71.40785217285156, 105.845458984375, 32.0539665222168, 17.519691467285156, 549.3359375, 492.8629150390625, 636.0932006835938]], "relative_norm": [[0.002082162071019411, 0.1830902397632599, 0.04237836226820946, 0.14487667381763458, 0.006035586819052696, 0.16686125099658966, 0.0016011283732950687, 0.07198508828878403, 0.11035122722387314, 0.03061671555042267, 0.014374860562384129, 0.5918878316879272, 0.4385133683681488, 0.61829674243927]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.5096435546875, 31.316137313842773, 8.941094398498535, 539.4290161132812, 490.3675842285156, 632.1050415039062]], "actual_norm": [[32.9312744140625, 142.42698669433594, 47.13922882080078, 163.4822235107422, 6.057154178619385, 182.66525268554688, 4.481956958770752, 66.53463745117188, 105.5003433227539, 31.346784591674805, 9.08966064453125, 539.3757934570312, 490.3880615234375, 632.1666870117188]], "relative_norm": [[0.002082162071019411, 0.18197228014469147, 0.04080353304743767, 0.14443343877792358, 0.005695999134331942, 0.16927148401737213, 0.0043270704336464405, 0.06686882674694061, 0.1097656860947609, 0.030022328719496727, 0.007452928461134434, 0.5817808508872986, 0.4376298785209656, 0.615807056427002]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 27602, 393, 3651, 19461, 1471, 422, 43680, 7513, 17672], "first_top_logits": [26.0, 25.125, 25.0, 21.125, 20.875, 20.25, 19.875, 19.875, 19.625, 19.5], "source_correct": true, "target_correct": false, "seconds": 0.4254942499101162, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1359537839889526, "target_logp": -11.769268035888672, "log_odds": -10.63331425189972, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.10562852770090103, -1.0186326503753662, -0.011458184570074081, -0.00023445718397852033]}, "target": {"ids": [56769, 369, 396], "log_probs": [-11.543128967285156, -0.2258150279521942, -0.0003244350664317608]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.392578125, 48.985652923583984, 164.44931030273438, 6.291029930114746, 180.39573669433594, 1.6272830963134766, 71.39640808105469, 105.84648895263672, 32.00467300415039, 17.41713523864746, 549.32763671875, 492.88006591796875, 636.0215454101562]], "actual_norm": [[32.919979095458984, 145.3661651611328, 48.982269287109375, 164.5002899169922, 6.37715482711792, 180.38011169433594, 1.6790040731430054, 71.34098815917969, 105.8484115600586, 32.06821060180664, 17.48346519470215, 549.2459106445312, 492.80615234375, 635.993408203125]], "relative_norm": [[0.002081447746604681, 0.18300239741802216, 0.04237993061542511, 0.14490282535552979, 0.006004463415592909, 0.1668451428413391, 0.0016169911250472069, 0.07191768288612366, 0.11035430431365967, 0.03063032031059265, 0.014345137402415276, 0.5917907953262329, 0.43846288323402405, 0.6181997060775757]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.392578125, 48.985652923583984, 164.44931030273438, 6.291029930114746, 180.39573669433594, 1.6272830963134766, 71.39640808105469, 105.84648895263672, 32.00467300415039, 17.41713523864746, 549.32763671875, 492.88006591796875, 636.0215454101562]], "actual_norm": [[32.919979095458984, 145.3661651611328, 48.982269287109375, 164.5002899169922, 6.37715482711792, 180.38011169433594, 1.6790040731430054, 71.34098815917969, 105.8484115600586, 32.06821060180664, 17.48346519470215, 549.2459106445312, 492.80615234375, 635.993408203125]], "relative_norm": [[0.002081447746604681, 0.18300239741802216, 0.04237993061542511, 0.14490282535552979, 0.006004463415592909, 0.1668451428413391, 0.0016169911250472069, 0.07191768288612366, 0.11035430431365967, 0.03063032031059265, 0.014345137402415276, 0.5917907953262329, 0.43846288323402405, 0.6181997060775757]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131103515625, 163.45777893066406, 5.96571159362793, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.50963592529297, 31.31613540649414, 8.941094398498535, 539.428955078125, 490.3675537109375, 632.1050415039062]], "actual_norm": [[32.919979095458984, 142.45469665527344, 47.12263107299805, 163.44210815429688, 6.064997673034668, 182.65150451660156, 4.511297225952148, 66.51761627197266, 105.5612564086914, 31.328733444213867, 9.108593940734863, 539.3438720703125, 490.3333435058594, 632.1181640625]], "relative_norm": [[0.002081447746604681, 0.18200768530368805, 0.0407891683280468, 0.14439800381660461, 0.005703375209122896, 0.1692587435245514, 0.00435539660975337, 0.06685172021389008, 0.10982906818389893, 0.030005039647221565, 0.007468452677130699, 0.5817463994026184, 0.43758106231689453, 0.6157597899436951]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 1863, 19461, 17672, 58068, 112020, 43680, 1147, 7513, 220], "first_top_logits": [24.75, 22.125, 21.75, 21.0, 19.5, 18.75, 18.25, 18.125, 18.0, 17.875], "source_correct": true, "target_correct": false, "seconds": 0.4352204157039523, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.4505348205566406, "target_logp": -13.393584251403809, "log_odds": -11.943049430847168, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.12038158625364304, -1.31534743309021, -0.014489431865513325, -0.0003163314249832183]}, "target": {"ids": [56769, 369, 396], "log_probs": [-13.182881355285645, -0.21029749512672424, -0.00040487200021743774]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.97646713256836, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.395751953125, 1.6272832155227661, 71.39642333984375, 105.84649658203125, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.880126953125, 636.0215454101562]], "actual_norm": [[32.968055725097656, 145.35513305664062, 48.96630859375, 164.46690368652344, 6.37249231338501, 180.36148071289062, 1.675033450126648, 71.3723373413086, 105.8109130859375, 32.01997756958008, 17.487102508544922, 549.37646484375, 492.96826171875, 636.0498657226562]], "relative_norm": [[0.002084487583488226, 0.1829885095357895, 0.04236612096428871, 0.14487341046333313, 0.0060000731609761715, 0.16682791709899902, 0.0016131671145558357, 0.07194928824901581, 0.11031521111726761, 0.030584249645471573, 0.014348121359944344, 0.5919314622879028, 0.4386070966720581, 0.6182546019554138]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.97646713256836, 145.39259338378906, 48.98565673828125, 164.44932556152344, 6.291030406951904, 180.395751953125, 1.6272832155227661, 71.39642333984375, 105.84649658203125, 32.00467300415039, 17.417137145996094, 549.3276977539062, 492.880126953125, 636.0215454101562]], "actual_norm": [[32.968055725097656, 145.35513305664062, 48.96630859375, 164.46690368652344, 6.37249231338501, 180.36148071289062, 1.675033450126648, 71.3723373413086, 105.8109130859375, 32.01997756958008, 17.487102508544922, 549.37646484375, 492.96826171875, 636.0498657226562]], "relative_norm": [[0.002084487583488226, 0.1829885095357895, 0.04236612096428871, 0.14487341046333313, 0.0060000731609761715, 0.16682791709899902, 0.0016131671145558357, 0.07194928824901581, 0.11031521111726761, 0.030584249645471573, 0.014348121359944344, 0.5919314622879028, 0.4386070966720581, 0.6182546019554138]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.97646713256836, 142.39881896972656, 47.131107330322266, 163.4578094482422, 5.96571159362793, 182.67059326171875, 4.409332275390625, 66.5072021484375, 105.5096435546875, 31.316139221191406, 8.941094398498535, 539.4290161132812, 490.36761474609375, 632.1050415039062]], "actual_norm": [[32.968055725097656, 142.43861389160156, 47.145652770996094, 163.4409942626953, 6.10033655166626, 182.65121459960938, 4.498037815093994, 66.49507904052734, 105.4824447631836, 31.338075637817383, 9.081182479858398, 539.3773193359375, 490.31939697265625, 632.0418090820312]], "relative_norm": [[0.002084487583488226, 0.18198713660240173, 0.04080909490585327, 0.14439702033996582, 0.005736606661230326, 0.1692584753036499, 0.0043425955809652805, 0.06682907044887543, 0.10974706709384918, 0.03001398779451847, 0.007445977069437504, 0.5817825198173523, 0.4375686049461365, 0.6156854033470154]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 17672, 19461, 393, 58068, 18050, 7513, 112020, 13, 43680], "first_top_logits": [24.0, 21.75, 20.75, 19.5, 17.75, 17.25, 17.125, 17.0, 16.875, 16.375], "source_correct": true, "target_correct": false, "seconds": 0.47164395777508616, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-20/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-20", "split": "eval", "country": "Poland", "currency": "Polish zloty", "aliases": ["zloty", "PLN", "z\u0142oty"], "city": "Gdansk", "target_country": "Hungary", "target_currency": "Hungarian forint", "target_aliases": ["forint", "HUF"], "prompt": "Fact: The currency used in the country containing Gdansk is the", "source_continuation": " Polish zloty", "target_continuation": " Hungarian forint"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.9781973361968994, "target_logp": -12.081439971923828, "log_odds": -11.103242635726929, "token_scores": {"source": {"ids": [31984, 1147, 9184, 88], "log_probs": [-0.19717074930667877, -0.7677568197250366, -0.013099211268126965, -0.0001705739414319396]}, "target": {"ids": [56769, 369, 396], "log_probs": [-11.884671211242676, -0.19641810655593872, -0.0003511289251036942]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44931030273438, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.41713523864746, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.92869567871094, 145.38853454589844, 48.972450256347656, 164.42532348632812, 6.3936262130737305, 180.3600311279297, 1.6585209369659424, 71.43326568603516, 105.8650131225586, 32.00264358520508, 17.52983856201172, 549.3790283203125, 492.9425354003906, 635.9462280273438]], "relative_norm": [[0.0020819988567382097, 0.18303056061267853, 0.04237143322825432, 0.14483678340911865, 0.006019971799105406, 0.16682657599449158, 0.0015972645487636328, 0.07201071083545685, 0.11037161946296692, 0.03056769259274006, 0.014383186586201191, 0.5919342637062073, 0.43858420848846436, 0.6181538701057434]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 145.39259338378906, 48.98565673828125, 164.44931030273438, 6.291030406951904, 180.39573669433594, 1.6272832155227661, 71.39641571044922, 105.84648895263672, 32.00467300415039, 17.41713523864746, 549.3276977539062, 492.8800964355469, 636.0215454101562]], "actual_norm": [[32.92869567871094, 145.38853454589844, 48.972450256347656, 164.42532348632812, 6.3936262130737305, 180.3600311279297, 1.6585209369659424, 71.43326568603516, 105.8650131225586, 32.00264358520508, 17.52983856201172, 549.3790283203125, 492.9425354003906, 635.9462280273438]], "relative_norm": [[0.0020819988567382097, 0.18303056061267853, 0.04237143322825432, 0.14483678340911865, 0.006019971799105406, 0.16682657599449158, 0.0015972645487636328, 0.07201071083545685, 0.11037161946296692, 0.03056769259274006, 0.014383186586201191, 0.5919342637062073, 0.43858420848846436, 0.6181538701057434]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Polish Zloty.  What else do we know about", "output_ids": [31984, 1863, 9184, 88, 13, 220, 3555, 770, 653, 582, 1414, 911], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 479, 67, 40774, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[32.976463317871094, 142.3988037109375, 47.131107330322266, 163.45779418945312, 5.9657111167907715, 182.6705780029297, 4.409332275390625, 66.5072021484375, 105.50963592529297, 31.31613540649414, 8.941094398498535, 539.428955078125, 490.3675537109375, 632.1050415039062]], "actual_norm": [[32.92869567871094, 142.43060302734375, 47.1179313659668, 163.47079467773438, 6.083796977996826, 182.6687774658203, 4.489026069641113, 66.54058837890625, 105.51892852783203, 31.323360443115234, 9.072842597961426, 539.3948364257812, 490.44390869140625, 632.0261840820312]], "relative_norm": [[0.0020819988567382097, 0.18197689950466156, 0.04078510031104088, 0.14442335069179535, 0.005721053574234247, 0.16927474737167358, 0.004333895165473223, 0.06687480956315994, 0.10978502780199051, 0.02999989315867424, 0.00743913883343339, 0.5818014144897461, 0.4376797378063202, 0.6156702041625977]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [31984, 1863, 19461, 43680, 17672, 5938, 479, 422, 30109, 58068], "first_top_logits": [23.5, 21.5, 20.625, 19.25, 18.875, 18.5, 18.0, 18.0, 17.625, 17.5], "source_correct": true, "target_correct": false, "seconds": 0.4655529581941664, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/readout", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 47.5, "target_rank": 9295.5, "source_logit": 5.625, "target_logit": -0.7890625, "top_ids": [8330, 38352, 7513, 47238, 6364, 31491, 45418, 19312, 4505, 2283, 46427, 1111], "top_tokens": [" Australia", "uenta", " European", " Regulation", " English", "uesto", ".currency", "urname", " Europe", "ential", "/is", "ule"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 349.0, "target_rank": 2710.0, "source_logit": 4.0, "target_logit": 1.3515625, "top_ids": [11413, 15973, 96694, 3001, 28453, 3693, 7065, 18174, 5088, 6321, 53661, 15154], "top_tokens": [" currency", "currency", "onis", "ica", " Currency", " American", "ican", "oret", "urrency", "si", "cla", " Spanish"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 410.0, "target_rank": 1283.0, "source_logit": 13.125, "target_logit": 10.25, "top_ids": [11413, 35602, 3263, 22956, 32488, 36911, 10465, 7513, 90475, 8330, 85466, 12463], "top_tokens": [" currency", " currencies", "\".", "\".\n", " euros", ".\".", ".\",", " European", "(currency", " Australia", " Euros", " colour"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 49.0, "target_rank": 312.5, "source_logit": 15.9375, "target_logit": 13.25, "top_ids": [11413, 7214, 13224, 35602, 47412, 65001, 7855, 32488, 21130, 74508, 9625, 3220], "top_tokens": [" currency", " foreign", " Britain", " currencies", " foreigners", "\"\u2014", " British", " euros", " parliament", "\u00bb.", " France", " money"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 4930.0, "target_rank": 3915.5, "source_logit": 0.361328125, "target_logit": 0.6953125, "top_ids": [3776, 21251, 3731, 12751, 11426, 5737, 603, 76231, 9127, 4229, 55964, 10948], "top_tokens": [" One", "onia", " Red", " Roman", " Land", " Group", "one", " Maritime", " Rock", " Ge", "\u2011", " River"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1002.0, "target_rank": 3554.5, "source_logit": 2.265625, "target_logit": 0.443359375, "top_ids": [7179, 364, 11766, 11413, 2932, 479, 6489, 18174, 29991, 547, 64, 4716], "top_tokens": [" International", " '", " Standard", " currency", " She", " G", " international", "oret", "\u540d\u79f0", " U", "a", " Bar"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 5521.5, "target_rank": 5013.5, "source_logit": 10.75, "target_logit": 10.9375, "top_ids": [11413, 13867, 35602, 30176, 29973, 91984, 3146, 6290, 3283, 5313, 7576, 72836], "top_tokens": [" currency", " county", " currencies", " counties", " licence", " licences", " country", " town", " city", " national", " prices", " postcode"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 24.5, "target_rank": 739.5, "source_logit": 17.375, "target_logit": 12.6875, "top_ids": [11413, 7513, 35602, 17672, 5837, 94936, 6489, 7855, 7214, 4505, 32488, 6364], "top_tokens": [" currency", " European", " currencies", " euro", " countries", " denomination", " international", " British", " foreign", " Europe", " euros", " English"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 12439.5, "target_rank": 3918.0, "source_logit": -1.546875, "target_logit": 0.447265625, "top_ids": [4229, 3693, 4787, 6364, 279, 3639, 2240, 89478, 271, 7855, 46427, 5434], "top_tokens": [" Ge", " American", " North", " English", " the", " United", " None", "\u2026\n", "\n\n", " British", "/is", "\u2026\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 3706.5, "target_rank": 821.5, "source_logit": 0.34765625, "target_logit": 2.5625, "top_ids": [1852, 5938, 3639, 3693, 4102, 271, 4787, 2701, 7179, 479, 5055, 6364], "top_tokens": [" same", " German", " United", " American", "\u00a0", "\n\n", " North", " following", " International", " G", " National", " English"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 5156.0, "target_rank": 32710.5, "source_logit": 5.1875, "target_logit": 0.59375, "top_ids": [104142, 103408, 11413, 35602, 113883, 112455, 28453, 32488, 110546, 90475, 26321, 106333], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", " currency", " currencies", "\u672c\u56fd", "\u82f1\u9551", " Currency", " euros", "\u82f1\u683c\u5170", "(currency", "Currency", "\u5916\u6c47"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 264.5, "target_rank": 13676.0, "source_logit": 7.34375, "target_logit": -0.2451171875, "top_ids": [104142, 103408, 106333, 113883, 110065, 106127, 112397, 101406, 116866, 109442, 101237, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u5916\u6c47", "\u672c\u56fd", "\u6c47\u7387", "\u6b27\u76df", "\u56fd\u5bb6\u6807\u51c6", "\u8d27\u5e01", "\u76f4\u8f96\u5e02", "\u8d22\u653f\u90e8", "\u7f8e\u5143", "\u6210\u5458\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 27.0, "target_rank": 1719.5, "source_logit": 16.5, "target_logit": 4.34375, "top_ids": [11413, 32488, 85466, 103408, 104142, 37088, 11192, 112455, 17672, 28453, 19551, 56769], "top_tokens": [" currency", " euros", " Euros", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " EUR", " dollars", "\u82f1\u9551", " euro", " Currency", " USD", " Hungarian"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 8.5, "target_rank": 573.0, "source_logit": 19.75, "target_logit": 6.1875, "top_ids": [56769, 11413, 32488, 104142, 17672, 103408, 85466, 37088, 42776, 19551, 35602, 17692], "top_tokens": [" Hungarian", " currency", " euros", "\u4eba\u6c11\u5e01", " euro", "\u6b27\u5143", " Euros", " EUR", " Hungary", " USD", " currencies", " dollar"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 31.0, "target_rank": 2335.0, "source_logit": 25.75, "target_logit": 7.125, "top_ids": [103408, 32488, 11413, 104142, 35602, 112455, 85466, 17672, 11192, 28453, 37088, 17692], "top_tokens": ["\u6b27\u5143", " euros", " currency", "\u4eba\u6c11\u5e01", " currencies", "\u82f1\u9551", " Euros", " euro", " dollars", " Currency", " EUR", " dollar"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 8.0, "target_rank": 2494.0, "source_logit": 24.375, "target_logit": 4.59375, "top_ids": [103408, 104142, 32488, 56769, 35602, 17672, 11413, 42776, 37088, 85466, 112455, 101406], "top_tokens": ["\u6b27\u5143", "\u4eba\u6c11\u5e01", " euros", " Hungarian", " currencies", " euro", " currency", " Hungary", " EUR", " Euros", "\u82f1\u9551", "\u8d27\u5e01"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 17.0, "target_rank": 5063.5, "source_logit": 18.25, "target_logit": 5.15625, "top_ids": [279, 56769, 472, 19461, 17672, 2598, 37088, 468, 264, 7513, 3881, 32488], "top_tokens": [" the", " Hungarian", " H", " Euro", " euro", " called", " EUR", " E", " a", " European", " known", " euros"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 9.5, "target_rank": 8546.5, "source_logit": 18.625, "target_logit": 3.015625, "top_ids": [56769, 17672, 19461, 472, 7513, 1752, 37088, 369, 89375, 42776, 468, 9812], "top_tokens": [" Hungarian", " euro", " Euro", " H", " European", " For", " EUR", " for", "Hung", " Hungary", " E", " EU"]}], "source_token": 42776, "target_token": 6778, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Deb", "rec", "en", " is", " the"], "seconds": 1.1360087497159839}
{"run_id": "eval-21/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6587728261947632, "target_logp": -21.693191528320312, "log_odds": -21.03441870212555, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.35053837299346924, -0.3080597221851349, -0.00017474555352237076]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-20.38178825378418, -1.3098512887954712, -0.001551734865643084]}}, "score_trace": {"source": [], "target": []}, "output": " Hungarian forint.  What else do we know about Deb", "output_ids": [56769, 369, 396, 13, 220, 3555, 770, 653, 582, 1414, 911, 35461], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [], "first_top_ids": [56769, 17672, 19461, 472, 7513, 1752, 37088, 369, 89375, 42776], "first_top_logits": [26.0, 25.0, 24.125, 22.375, 21.125, 20.5, 20.25, 20.0, 18.625, 18.625], "source_correct": true, "target_correct": false, "seconds": 0.45004258304834366, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6587728261947632, "target_logp": -21.693191528320312, "log_odds": -21.03441870212555, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.35053837299346924, -0.3080597221851349, -0.00017474555352237076]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-20.38178825378418, -1.3098512887954712, -0.001551734865643084]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.  What else do we know about Deb", "output_ids": [56769, 369, 396, 13, 220, 3555, 770, 653, 582, 1414, 911, 35461], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 17672, 19461, 472, 7513, 1752, 37088, 369, 89375, 42776], "first_top_logits": [26.0, 25.0, 24.125, 22.375, 21.125, 20.5, 20.25, 20.0, 18.625, 18.625], "source_correct": true, "target_correct": false, "seconds": 0.4386972077190876, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -38.01417541503906, "target_logp": -5.275266647338867, "log_odds": 32.738908767700195, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-37.737525939941406, -0.27646926045417786, -0.00017963226127903908]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-4.018774509429932, -1.2552739381790161, -0.0012184107908979058]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 187.02809143066406, 934.9352416992188, 683.6514282226562, 855.0044555664062, 1001.1445922851562]], "actual_norm": [[237.18377685546875, 0.12470667064189911, 16.22233009338379, 181.93910217285156, 137.443603515625, 228.77517700195312, 30.124174118041992, 227.0650177001953, 126.4953384399414, 187.08169555664062, 934.99072265625, 683.6546630859375, 855.0269775390625, 1001.0891723632812]], "relative_norm": [[0.014996536076068878, 0.00015699403593316674, 0.014035715721547604, 0.16026408970355988, 0.12941117584705353, 0.21160884201526642, 0.02901155687868595, 0.22890053689479828, 0.13188016414642334, 0.13699683547019958, 0.7742835283279419, 0.7336052656173706, 0.7865502238273621, 0.9553657174110413]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 187.02809143066406, 934.9352416992188, 683.6514282226562, 855.0044555664062, 1001.1445922851562]], "actual_norm": [[237.18377685546875, 0.12470667064189911, 16.22233009338379, 181.93910217285156, 137.443603515625, 228.77517700195312, 30.124174118041992, 227.0650177001953, 126.4953384399414, 187.08169555664062, 934.99072265625, 683.6546630859375, 855.0269775390625, 1001.0891723632812]], "relative_norm": [[0.014996536076068878, 0.00015699403593316674, 0.014035715721547604, 0.16026408970355988, 0.12941117584705353, 0.21160884201526642, 0.02901155687868595, 0.22890053689479828, 0.13188016414642334, 0.13699683547019958, 0.7742835283279419, 0.7336052656173706, 0.7865502238273621, 0.9553657174110413]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israel New Shekel.\n\nFact: The currency used in the", "output_ids": [6778, 1532, 2932, 17659, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 181.52455139160156, 927.6989135742188, 676.1608276367188, 852.9365234375, 1001.753662109375]], "actual_norm": [[237.18377685546875, 0.0527481883764267, 20.993175506591797, 180.9279327392578, 136.81155395507812, 227.3384246826172, 33.47581100463867, 222.78677368164062, 125.27589416503906, 181.54368591308594, 927.7694702148438, 676.16552734375, 852.92431640625, 1001.7026977539062]], "relative_norm": [[0.014996536076068878, 6.739388481946662e-05, 0.018171612173318863, 0.15984639525413513, 0.12865422666072845, 0.21066902577877045, 0.03231896087527275, 0.2239057868719101, 0.13034075498580933, 0.1331809163093567, 0.7698603272438049, 0.7291563153266907, 0.786367654800415, 0.9537732601165771]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6778, 15907, 17692, 1852, 13350, 320, 1584, 5313, 2118, 4741], "first_top_logits": [32.5, 28.375, 24.875, 22.375, 21.875, 20.75, 20.625, 20.125, 19.75, 19.75], "source_correct": false, "target_correct": true, "seconds": 0.43953562481328845, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -7.634223461151123, "target_logp": -21.089481353759766, "log_odds": -13.455257892608643, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-7.269866943359375, -0.36421602964401245, -0.00014077626110520214]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-19.707366943359375, -1.380562663078308, -0.0015524489572271705]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.15806007385254, 227.02777099609375, 126.50897979736328, 187.02809143066406, 934.935302734375, 683.6514282226562, 855.0044555664062, 1001.1446533203125]], "actual_norm": [[237.25454711914062, 0.14219249784946442, 16.188480377197266, 181.866943359375, 137.3607635498047, 228.80654907226562, 30.174673080444336, 227.0655517578125, 126.51718139648438, 187.04684448242188, 935.00732421875, 683.6950073242188, 854.898193359375, 1001.1360473632812]], "relative_norm": [[0.015001010149717331, 0.00017900705279316753, 0.01400642842054367, 0.16020052134990692, 0.12933316826820374, 0.21163786947727203, 0.02906019054353237, 0.22890107333660126, 0.13190293312072754, 0.13697132468223572, 0.7742972373962402, 0.7336485385894775, 0.7864317893981934, 0.9554104804992676]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.15806007385254, 227.02777099609375, 126.50897979736328, 187.02809143066406, 934.935302734375, 683.6514282226562, 855.0044555664062, 1001.1446533203125]], "actual_norm": [[237.25454711914062, 0.14219249784946442, 16.188480377197266, 181.866943359375, 137.3607635498047, 228.80654907226562, 30.174673080444336, 227.0655517578125, 126.51718139648438, 187.04684448242188, 935.00732421875, 683.6950073242188, 854.898193359375, 1001.1360473632812]], "relative_norm": [[0.015001010149717331, 0.00017900705279316753, 0.01400642842054367, 0.16020052134990692, 0.12933316826820374, 0.21163786947727203, 0.02906019054353237, 0.22890107333660126, 0.13190293312072754, 0.13697132468223572, 0.7742972373962402, 0.7336485385894775, 0.7864317893981934, 0.9554104804992676]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Deb", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 35461], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.9359188079834, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 181.52455139160156, 927.6988525390625, 676.1608276367188, 852.9364624023438, 1001.753662109375]], "actual_norm": [[237.25454711914062, 0.053359173238277435, 20.947769165039062, 180.89378356933594, 136.80953979492188, 227.40267944335938, 33.44046401977539, 222.81056213378906, 125.3204116821289, 181.4835205078125, 927.7557373046875, 676.0938110351562, 852.9728393554688, 1001.7885131835938]], "relative_norm": [[0.015001010149717331, 6.817451503593475e-05, 0.01813230849802494, 0.1598162204027176, 0.12865233421325684, 0.21072857081890106, 0.03228483349084854, 0.22392968833446503, 0.130387082695961, 0.1331367790699005, 0.7698489427566528, 0.729079008102417, 0.7864124178886414, 0.9538549780845642]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 17672, 85466, 19461, 7513, 56769, 37088, 32488, 6633, 220], "first_top_logits": [28.75, 23.875, 23.75, 23.5, 23.0, 22.0, 21.375, 21.125, 20.375, 20.25], "source_correct": false, "target_correct": false, "seconds": 0.42081975005567074, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -7.246774196624756, "target_logp": -51.46727752685547, "log_odds": -44.22050333023071, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-6.876116752624512, -0.3704703450202942, -0.0001867835089797154]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-50.25111770629883, -1.2144109010696411, -0.0017463208641856909]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 187.02809143066406, 934.9352416992188, 683.6514282226562, 855.0044555664062, 1001.1445922851562]], "actual_norm": [[237.26779174804688, 0.12294193357229233, 16.219467163085938, 181.9257049560547, 137.43594360351562, 228.81753540039062, 30.143882751464844, 226.9967041015625, 126.51493835449219, 187.05970764160156, 934.935546875, 683.5892333984375, 855.0047607421875, 1001.1854248046875]], "relative_norm": [[0.015001848340034485, 0.00015477239503525198, 0.014033238403499126, 0.16025228798389435, 0.1294039636850357, 0.21164803206920624, 0.029030537232756615, 0.2288316786289215, 0.1319005936384201, 0.13698074221611023, 0.7742378115653992, 0.7335350513458252, 0.7865298390388489, 0.9554575681686401]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 187.02809143066406, 934.9352416992188, 683.6514282226562, 855.0044555664062, 1001.1445922851562]], "actual_norm": [[237.26779174804688, 0.12294193357229233, 16.219467163085938, 181.9257049560547, 137.43594360351562, 228.81753540039062, 30.143882751464844, 226.9967041015625, 126.51493835449219, 187.05970764160156, 934.935546875, 683.5892333984375, 855.0047607421875, 1001.1854248046875]], "relative_norm": [[0.015001848340034485, 0.00015477239503525198, 0.014033238403499126, 0.16025228798389435, 0.1294039636850357, 0.21164803206920624, 0.029030537232756615, 0.2288316786289215, 0.1319005936384201, 0.13698074221611023, 0.7742378115653992, 0.7335350513458252, 0.7865298390388489, 0.9554575681686401]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungary forint.  What else do we know about Deb", "output_ids": [42776, 369, 396, 13, 220, 3555, 770, 653, 582, 1414, 911, 35461], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 181.52455139160156, 927.6989135742188, 676.1608276367188, 852.9365234375, 1001.753662109375]], "actual_norm": [[237.26779174804688, 0.0520724318921566, 20.986572265625, 180.92987060546875, 136.83164978027344, 227.33685302734375, 33.4898567199707, 222.83822631835938, 125.26361846923828, 181.5059356689453, 927.7011108398438, 676.1155395507812, 852.8973999023438, 1001.5982666015625]], "relative_norm": [[0.015001848340034485, 6.653050513705239e-05, 0.018165895715355873, 0.15984810888767242, 0.12867312133312225, 0.21066758036613464, 0.03233252093195915, 0.22395749390125275, 0.1303279846906662, 0.13315322995185852, 0.7698035836219788, 0.7291024327278137, 0.7863428592681885, 0.9536738395690918]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [42776, 56769, 103670, 89375, 25523, 18295, 472, 69595, 11278, 12200], "first_top_logits": [30.0, 23.25, 20.25, 19.625, 18.125, 16.75, 15.625, 15.5, 13.6875, 13.125], "source_correct": true, "target_correct": false, "seconds": 0.4681220422498882, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.730390548706055, "target_logp": -18.841270446777344, "log_odds": -14.110879898071289, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-4.285910129547119, -0.4443337321281433, -0.00014673586701974273]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-17.53590965270996, -1.3028210401535034, -0.002540814923122525]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.158061981201172, 227.0277557373047, 126.50896453857422, 187.028076171875, 934.9351806640625, 683.6514282226562, 855.00439453125, 1001.1445922851562]], "actual_norm": [[236.99099731445312, 0.12726791203022003, 16.17294692993164, 181.87330627441406, 137.4217529296875, 228.81961059570312, 30.24599838256836, 226.97975158691406, 126.50064086914062, 187.074951171875, 934.9293823242188, 683.8038330078125, 855.0217895507812, 1001.1837768554688]], "relative_norm": [[0.01498434692621231, 0.00016021839110180736, 0.013992988504469395, 0.1602061241865158, 0.12939059734344482, 0.21164995431900024, 0.02912888117134571, 0.22881458699703217, 0.13188569247722626, 0.13699190318584442, 0.7742326855659485, 0.7337653040885925, 0.78654545545578, 0.955456018447876]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.158061981201172, 227.0277557373047, 126.50896453857422, 187.028076171875, 934.9351806640625, 683.6514282226562, 855.00439453125, 1001.1445922851562]], "actual_norm": [[236.99099731445312, 0.12726791203022003, 16.17294692993164, 181.87330627441406, 137.4217529296875, 228.81961059570312, 30.24599838256836, 226.97975158691406, 126.50064086914062, 187.074951171875, 934.9293823242188, 683.8038330078125, 855.0217895507812, 1001.1837768554688]], "relative_norm": [[0.01498434692621231, 0.00016021839110180736, 0.013992988504469395, 0.1602061241865158, 0.12939059734344482, 0.21164995431900024, 0.02912888117134571, 0.22881458699703217, 0.13188569247722626, 0.13699190318584442, 0.7742326855659485, 0.7337653040885925, 0.78654545545578, 0.955456018447876]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  What is the currency used in the country containing", "output_ids": [17672, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.1332213282585144, 20.935914993286133, 180.9315948486328, 136.80807495117188, 227.33010864257812, 33.43550109863281, 222.82386779785156, 125.29711151123047, 181.5245361328125, 927.6988525390625, 676.1607666015625, 852.9364624023438, 1001.7536010742188]], "actual_norm": [[236.99099731445312, 0.0524776317179203, 20.942527770996094, 180.89088439941406, 136.83682250976562, 227.3363800048828, 33.53871154785156, 222.7841033935547, 125.32426452636719, 181.47628784179688, 927.7325439453125, 676.1885375976562, 852.9249267578125, 1001.6781005859375]], "relative_norm": [[0.01498434692621231, 6.704821134917438e-05, 0.018127771094441414, 0.15981365740299225, 0.12867799401283264, 0.21066713333129883, 0.03237968683242798, 0.2239031046628952, 0.13039109110832214, 0.1331314742565155, 0.7698296904563904, 0.729181170463562, 0.7863682508468628, 0.9537498354911804]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 56769, 468, 472, 7513, 85466, 32488, 37088, 103408], "first_top_logits": [25.25, 24.375, 21.25, 19.625, 19.25, 19.25, 18.75, 18.375, 17.625, 17.25], "source_correct": false, "target_correct": false, "seconds": 0.43815708300098777, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.3343145847320557, "target_logp": -18.303443908691406, "log_odds": -14.96912932395935, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-3.0320136547088623, -0.3021217882633209, -0.00017915551143232733]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-16.938262939453125, -1.3634251356124878, -0.0017563169822096825]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.8834686279297, 137.37698364257812, 228.79539489746094, 30.158063888549805, 227.02777099609375, 126.50897979736328, 187.02810668945312, 934.935302734375, 683.6514282226562, 855.0045166015625, 1001.1446533203125]], "actual_norm": [[237.1091766357422, 0.13315518200397491, 16.20315933227539, 181.87828063964844, 137.41062927246094, 228.7781524658203, 30.143505096435547, 226.99298095703125, 126.49353790283203, 187.0972137451172, 934.8716430664062, 683.682373046875, 854.9461669921875, 1001.0810546875]], "relative_norm": [[0.01499181892722845, 0.00016762991435825825, 0.014019128866493702, 0.160210520029068, 0.12938012182712555, 0.21161159873008728, 0.029030174016952515, 0.22882792353630066, 0.13187828660011292, 0.1370082050561905, 0.7741848826408386, 0.7336349487304688, 0.7864758968353271, 0.9553579688072205]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.8834686279297, 137.37698364257812, 228.79539489746094, 30.158063888549805, 227.02777099609375, 126.50897979736328, 187.02810668945312, 934.935302734375, 683.6514282226562, 855.0045166015625, 1001.1446533203125]], "actual_norm": [[237.1091766357422, 0.13315518200397491, 16.20315933227539, 181.87828063964844, 137.41062927246094, 228.7781524658203, 30.143505096435547, 226.99298095703125, 126.49353790283203, 187.0972137451172, 934.8716430664062, 683.682373046875, 854.9461669921875, 1001.0810546875]], "relative_norm": [[0.01499181892722845, 0.00016762991435825825, 0.014019128866493702, 0.160210520029068, 0.12938012182712555, 0.21161159873008728, 0.029030174016952515, 0.22882792353630066, 0.13187828660011292, 0.1370082050561905, 0.7741848826408386, 0.7336349487304688, 0.7864758968353271, 0.9553579688072205]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  Inference: The country is in the European", "output_ids": [17672, 13, 220, 758, 2202, 25, 576, 3146, 374, 304, 279, 7513], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80810546875, 227.33013916015625, 33.43550491333008, 222.82388305664062, 125.297119140625, 181.52455139160156, 927.6989135742188, 676.1608276367188, 852.9365234375, 1001.7537231445312]], "actual_norm": [[237.1091766357422, 0.056986004114151, 20.9071102142334, 180.94696044921875, 136.82696533203125, 227.28517150878906, 33.41572189331055, 222.85386657714844, 125.28910827636719, 181.5026092529297, 927.75341796875, 676.215087890625, 852.9904174804688, 1001.8497314453125]], "relative_norm": [[0.01499181892722845, 7.280834688572213e-05, 0.01809711381793022, 0.1598632037639618, 0.12866872549057007, 0.21061968803405762, 0.0322609469294548, 0.22397321462631226, 0.13035450875759125, 0.13315078616142273, 0.76984703540802, 0.7292097806930542, 0.7864286303520203, 0.9539132714271545]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 37088, 56769, 13, 547, 10038, 32488, 3958, 892], "first_top_logits": [19.375, 18.5, 17.375, 17.125, 16.625, 16.5, 16.375, 16.375, 16.125, 16.125], "source_correct": false, "target_correct": false, "seconds": 0.4220990831963718, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-21/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-21", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Debrecen", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Debrecen is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.43258529901504517, "target_logp": -14.23196029663086, "log_odds": -13.799374997615814, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.2402324378490448, -0.19219349324703217, -0.00015937011630740017]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-13.365232467651367, -0.8651835322380066, -0.0015441172290593386]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.15806007385254, 227.02777099609375, 126.50896453857422, 187.028076171875, 934.9352416992188, 683.6514282226562, 855.00439453125, 1001.14453125]], "actual_norm": [[236.99652099609375, 0.13541673123836517, 16.213977813720703, 181.8157958984375, 137.3481903076172, 228.75857543945312, 30.200042724609375, 226.99557495117188, 126.52033996582031, 187.13169860839844, 934.8873291015625, 683.6957397460938, 854.9993286132812, 1001.1905517578125]], "relative_norm": [[0.014984696172177792, 0.00017047699657268822, 0.014028489589691162, 0.16015547513961792, 0.12932133674621582, 0.21159349381923676, 0.029084622859954834, 0.22883053123950958, 0.13190622627735138, 0.13703346252441406, 0.7741978764533997, 0.7336493134498596, 0.7865248322486877, 0.9554624557495117]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.15806007385254, 227.02777099609375, 126.50896453857422, 187.028076171875, 934.9352416992188, 683.6514282226562, 855.00439453125, 1001.14453125]], "actual_norm": [[236.99652099609375, 0.13541673123836517, 16.213977813720703, 181.8157958984375, 137.3481903076172, 228.75857543945312, 30.200042724609375, 226.99557495117188, 126.52033996582031, 187.13169860839844, 934.8873291015625, 683.6957397460938, 854.9993286132812, 1001.1905517578125]], "relative_norm": [[0.014984696172177792, 0.00017047699657268822, 0.014028489589691162, 0.16015547513961792, 0.12932133674621582, 0.21159349381923676, 0.029084622859954834, 0.22883053123950958, 0.13190622627735138, 0.13703346252441406, 0.7741978764533997, 0.7336493134498596, 0.7865248322486877, 0.9554624557495117]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.\n\nFact: The currency used in the country", "output_ids": [56769, 369, 396, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 35461, 2758, 268, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213282585144, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.33010864257812, 33.43550109863281, 222.82386779785156, 125.29711151123047, 181.52455139160156, 927.6988525390625, 676.1608276367188, 852.9364624023438, 1001.7536010742188]], "actual_norm": [[236.99652099609375, 0.05747038871049881, 20.956981658935547, 180.93589782714844, 136.800537109375, 227.3764190673828, 33.460350036621094, 222.86021423339844, 125.29371643066406, 181.52239990234375, 927.6421508789062, 676.1747436523438, 852.9121704101562, 1001.859375]], "relative_norm": [[0.014984696172177792, 7.342721801251173e-05, 0.018140282481908798, 0.15985342860221863, 0.12864387035369873, 0.2107042372226715, 0.032304033637046814, 0.22397959232330322, 0.13035930693149567, 0.13316529989242554, 0.7697547078132629, 0.7291662693023682, 0.786356508731842, 0.95392245054245]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 17672, 19461, 5938, 3639, 220, 444, 11623, 42364, 17692], "first_top_logits": [22.125, 20.625, 20.0, 17.375, 16.75, 16.625, 16.5, 15.8125, 15.5625, 15.375], "source_correct": true, "target_correct": false, "seconds": 0.48305045906454325, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-22/readout", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 20.0, "target_rank": 5068.0, "source_logit": 6.75, "target_logit": 0.279296875, "top_ids": [8330, 38352, 6364, 4505, 3001, 7513, 49029, 11413, 7065, 15973, 3693, 45418], "top_tokens": [" Australia", "uenta", " English", " Europe", "ica", " European", "ctica", " currency", "ican", "currency", " American", ".currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 216.5, "target_rank": 2437.5, "source_logit": 4.53125, "target_logit": 1.390625, "top_ids": [11413, 15973, 96694, 28453, 3001, 3693, 6364, 15154, 5088, 53661, 7065, 18174], "top_tokens": [" currency", "currency", "onis", " Currency", "ica", " American", " English", " Spanish", "urrency", "cla", "ican", "oret"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 128.0, "target_rank": 869.0, "source_logit": 14.625, "target_logit": 10.3125, "top_ids": [11413, 35602, 85466, 32488, 8330, 90475, 36911, 7513, 45418, 13369, 3263, 77186], "top_tokens": [" currency", " currencies", " Euros", " euros", " Australia", "(currency", ".\".", " European", ".currency", " Australian", "\".", ".Currency"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 45.0, "target_rank": 273.5, "source_logit": 17.125, "target_logit": 13.8125, "top_ids": [11413, 7214, 35602, 65001, 32488, 13224, 47412, 17672, 7855, 3220, 15973, 34710], "top_tokens": [" currency", " foreign", " currencies", "\"\u2014", " euros", " Britain", " foreigners", " euro", " British", " money", "currency", "_currency"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 3297.0, "target_rank": 2254.0, "source_logit": 0.9609375, "target_logit": 1.4765625, "top_ids": [21251, 28453, 55964, 3776, 3731, 7748, 21525, 4229, 603, 11413, 12751, 29921], "top_tokens": ["onia", " Currency", "\u2011", " One", " Red", " Indian", " Ring", " Ge", "one", " currency", " Roman", "\u2010"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 1698.0, "target_rank": 3904.0, "source_logit": 1.7734375, "target_logit": 0.48046875, "top_ids": [11413, 364, 2932, 11766, 7179, 18174, 479, 467, 28453, 6489, 4716, 105469], "top_tokens": [" currency", " '", " She", " Standard", " International", "oret", " G", " W", " Currency", " international", " Bar", "\u79cd\u7c7b"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 4359.5, "target_rank": 2905.5, "source_logit": 11.4375, "target_logit": 12.1875, "top_ids": [11413, 35602, 13867, 30176, 3146, 5837, 28453, 5313, 60261, 43895, 7214, 7576], "top_tokens": [" currency", " currencies", " county", " counties", " country", " countries", " Currency", " national", " tariff", " tariffs", " foreign", " prices"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 37.0, "target_rank": 812.5, "source_logit": 17.375, "target_logit": 12.875, "top_ids": [11413, 35602, 17672, 103408, 37198, 7513, 94936, 5837, 32488, 7855, 7214, 105492], "top_tokens": [" currency", " currencies", " euro", "\u6b27\u5143", " Commonwealth", " European", " denomination", " countries", " euros", " British", " foreign", "\u4e2d\u534e\u4eba\u6c11"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 12876.5, "target_rank": 3208.0, "source_logit": -1.6328125, "target_logit": 0.71875, "top_ids": [3693, 4229, 89478, 4787, 5434, 279, 220, 4102, 6364, 271, 2240, 12022], "top_tokens": [" American", " Ge", "\u2026\n", " North", "\u2026\n\n", " the", " ", "\u00a0", " English", "\n\n", " None", " \u2026\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 4095.5, "target_rank": 620.5, "source_logit": 0.197265625, "target_logit": 2.890625, "top_ids": [1852, 4102, 5938, 4787, 3693, 271, 3639, 4882, 479, 2274, 6364, 2701], "top_tokens": [" same", "\u00a0", " German", " North", " American", "\n\n", " United", " South", " G", " US", " English", " following"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 6971.5, "target_rank": 23073.5, "source_logit": 4.78125, "target_logit": 1.8515625, "top_ids": [104142, 11413, 35602, 103408, 113883, 28453, 26321, 32488, 106333, 101237, 90475, 115160], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " currencies", "\u6b27\u5143", "\u672c\u56fd", " Currency", "Currency", " euros", "\u5916\u6c47", "\u7f8e\u5143", "(currency", "\u5176\u4ed6\u56fd\u5bb6"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 576.0, "target_rank": 17242.0, "source_logit": 5.375, "target_logit": -0.8203125, "top_ids": [104142, 103408, 112397, 116866, 113883, 101237, 106127, 99599, 109442, 106333, 101406, 105492], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u672c\u56fd", "\u7f8e\u5143", "\u6b27\u76df", "\u56fd\u5bb6", "\u8d22\u653f\u90e8", "\u5916\u6c47", "\u8d27\u5e01", "\u4e2d\u534e\u4eba\u6c11"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 28.0, "target_rank": 1033.5, "source_logit": 16.5, "target_logit": 5.34375, "top_ids": [11413, 32488, 104142, 85466, 11192, 103408, 112455, 28453, 19551, 37088, 17672, 17692], "top_tokens": [" currency", " euros", "\u4eba\u6c11\u5e01", " Euros", " dollars", "\u6b27\u5143", "\u82f1\u9551", " Currency", " USD", " EUR", " euro", " dollar"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 7.0, "target_rank": 359.0, "source_logit": 20.0, "target_logit": 7.1875, "top_ids": [56769, 104142, 11413, 103408, 17672, 32488, 42776, 19551, 85466, 17692, 37088, 35602], "top_tokens": [" Hungarian", "\u4eba\u6c11\u5e01", " currency", "\u6b27\u5143", " euro", " euros", " Hungary", " USD", " Euros", " dollar", " EUR", " currencies"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 32.0, "target_rank": 1371.5, "source_logit": 27.625, "target_logit": 10.0, "top_ids": [103408, 11413, 104142, 32488, 112455, 35602, 11192, 28453, 85466, 17672, 17692, 80189], "top_tokens": ["\u6b27\u5143", " currency", "\u4eba\u6c11\u5e01", " euros", "\u82f1\u9551", " currencies", " dollars", " Currency", " Euros", " euro", " dollar", " Dollars"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 9.5, "target_rank": 1385.5, "source_logit": 28.0, "target_logit": 7.65625, "top_ids": [104142, 103408, 35602, 32488, 56769, 11413, 17672, 112455, 101237, 42776, 28453, 37088], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", " currencies", " euros", " Hungarian", " currency", " euro", "\u82f1\u9551", "\u7f8e\u5143", " Hungary", " Currency", " EUR"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 14.5, "target_rank": 4384.0, "source_logit": 18.75, "target_logit": 6.09375, "top_ids": [279, 56769, 472, 19461, 2598, 17672, 264, 7855, 369, 3881, 468, 1083], "top_tokens": [" the", " Hungarian", " H", " Euro", " called", " euro", " a", " British", " for", " known", " E", " also"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 11.0, "target_rank": 5337.0, "source_logit": 18.75, "target_logit": 4.625, "top_ids": [56769, 17672, 19461, 472, 1752, 7513, 369, 37088, 89375, 468, 42776, 444], "top_tokens": [" Hungarian", " euro", " Euro", " H", " For", " European", " for", " EUR", "Hung", " E", " Hungary", " L"]}], "source_token": 42776, "target_token": 6778, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Sz", "eg", "ed", " is", " the"], "seconds": 1.156631249934435}
{"run_id": "eval-22/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6550722122192383, "target_logp": -21.06064224243164, "log_odds": -20.405570030212402, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.20730674266815186, -0.4475571811199188, -0.00020823694649152458]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-18.863555908203125, -2.1958656311035156, -0.0012212683213874698]}}, "score_trace": {"source": [], "target": []}, "output": " Hungarian forint.\n\nAnother fact: The city of Szeg", "output_ids": [56769, 369, 396, 382, 14037, 2097, 25, 576, 3283, 315, 44190, 791], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [], "first_top_ids": [56769, 17672, 19461, 472, 1752, 7513, 369, 37088, 89375, 468], "first_top_logits": [26.25, 24.375, 23.625, 22.75, 21.5, 21.125, 21.0, 20.25, 19.0, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.4614581670612097, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-22/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6550722122192383, "target_logp": -21.06064224243164, "log_odds": -20.405570030212402, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.20730674266815186, -0.4475571811199188, -0.00020823694649152458]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-18.863555908203125, -2.1958656311035156, -0.0012212683213874698]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.\n\nAnother fact: The city of Szeg", "output_ids": [56769, 369, 396, 382, 14037, 2097, 25, 576, 3283, 315, 44190, 791], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 17672, 19461, 472, 1752, 7513, 369, 37088, 89375, 468], "first_top_logits": [26.25, 24.375, 23.625, 22.75, 21.5, 21.125, 21.0, 20.25, 19.0, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.46841879189014435, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-22/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -33.548038482666016, "target_logp": -6.013376235961914, "log_odds": 27.5346622467041, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-33.05696487426758, -0.49084386229515076, -0.0002274254511576146]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-3.775714159011841, -2.236811399459839, -0.0008505540899932384]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 635.0573120117188, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.2429809570312]], "actual_norm": [[237.18377685546875, 0.12470667064189911, 16.22233009338379, 181.93910217285156, 137.443603515625, 228.77517700195312, 30.124174118041992, 227.0650177001953, 126.4953384399414, 635.0816650390625, 1125.14892578125, 862.484375, 781.3688354492188, 935.1759643554688]], "relative_norm": [[0.014996536076068878, 0.00015699403593316674, 0.014035715721547604, 0.16026408970355988, 0.12941117584705353, 0.21160884201526642, 0.02901155687868595, 0.22890053689479828, 0.13188016414642334, 0.6024783849716187, 0.8696780204772949, 0.9054096341133118, 0.7127900123596191, 0.903453528881073]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 635.0573120117188, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.2429809570312]], "actual_norm": [[237.18377685546875, 0.12470667064189911, 16.22233009338379, 181.93910217285156, 137.443603515625, 228.77517700195312, 30.124174118041992, 227.0650177001953, 126.4953384399414, 635.0816650390625, 1125.14892578125, 862.484375, 781.3688354492188, 935.1759643554688]], "relative_norm": [[0.014996536076068878, 0.00015699403593316674, 0.014035715721547604, 0.16026408970355988, 0.12941117584705353, 0.21160884201526642, 0.02901155687868595, 0.22890053689479828, 0.13188016414642334, 0.6024783849716187, 0.8696780204772949, 0.9054096341133118, 0.7127900123596191, 0.903453528881073]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israel New Shekel.\n\nWhat is the country's name?\n\n", "output_ids": [6778, 1532, 2932, 17659, 382, 3838, 374, 279, 3146, 594, 829, 1939], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 625.5267333984375, 1130.1561279296875, 854.1030883789062, 784.0282592773438, 933.7200317382812]], "actual_norm": [[237.18377685546875, 0.0527481883764267, 20.993175506591797, 180.9279327392578, 136.81155395507812, 227.3384246826172, 33.47581100463867, 222.78677368164062, 125.27589416503906, 625.4757080078125, 1130.2076416015625, 854.0925903320312, 784.0601806640625, 933.7564697265625]], "relative_norm": [[0.014996536076068878, 6.739388481946662e-05, 0.018171612173318863, 0.15984639525413513, 0.12865422666072845, 0.21066902577877045, 0.03231896087527275, 0.2239057868719101, 0.13034075498580933, 0.5947322249412537, 0.8743916153907776, 0.8988814353942871, 0.7173312306404114, 0.9023901224136353]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6778, 15907, 17692, 1852, 5313, 13350, 320, 6995, 3639, 1584], "first_top_logits": [31.25, 27.5, 24.625, 22.5, 21.25, 21.0, 20.625, 20.0, 19.75, 19.75], "source_correct": false, "target_correct": true, "seconds": 0.4653660422191024, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-22/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.251278877258301, "target_logp": -18.527786254882812, "log_odds": -12.276507377624512, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-5.665001392364502, -0.5861024856567383, -0.00017510310863144696]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-16.290000915527344, -2.2364087104797363, -0.0013775869738310575]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.15806007385254, 227.02777099609375, 126.50897979736328, 635.0572509765625, 1125.1011962890625, 862.437255859375, 781.3258056640625, 935.2430419921875]], "actual_norm": [[237.25454711914062, 0.14219249784946442, 16.188480377197266, 181.866943359375, 137.3607635498047, 228.80654907226562, 30.174673080444336, 227.0655517578125, 126.51718139648438, 635.0537109375, 1125.0350341796875, 862.3939819335938, 781.2627563476562, 935.2823486328125]], "relative_norm": [[0.015001010149717331, 0.00017900705279316753, 0.01400642842054367, 0.16020052134990692, 0.12933316826820374, 0.21163786947727203, 0.02906019054353237, 0.22890107333660126, 0.13190293312072754, 0.6024518609046936, 0.8695899844169617, 0.9053147435188293, 0.7126932740211487, 0.9035562872886658]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.15806007385254, 227.02777099609375, 126.50897979736328, 635.0572509765625, 1125.1011962890625, 862.437255859375, 781.3258056640625, 935.2430419921875]], "actual_norm": [[237.25454711914062, 0.14219249784946442, 16.188480377197266, 181.866943359375, 137.3607635498047, 228.80654907226562, 30.174673080444336, 227.0655517578125, 126.51718139648438, 635.0537109375, 1125.0350341796875, 862.3939819335938, 781.2627563476562, 935.2823486328125]], "relative_norm": [[0.015001010149717331, 0.00017900705279316753, 0.01400642842054367, 0.16020052134990692, 0.12933316826820374, 0.21163786947727203, 0.02906019054353237, 0.22890107333660126, 0.13190293312072754, 0.6024518609046936, 0.8695899844169617, 0.9053147435188293, 0.7126932740211487, 0.9035562872886658]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Sz", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 44190], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.9359188079834, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 625.5267333984375, 1130.1561279296875, 854.1030883789062, 784.0281982421875, 933.719970703125]], "actual_norm": [[237.25454711914062, 0.053359173238277435, 20.947769165039062, 180.89378356933594, 136.80953979492188, 227.40267944335938, 33.44046401977539, 222.81056213378906, 125.3204116821289, 625.5382080078125, 1130.0567626953125, 854.012451171875, 783.9620361328125, 933.756591796875]], "relative_norm": [[0.015001010149717331, 6.817451503593475e-05, 0.01813230849802494, 0.1598162204027176, 0.12865233421325684, 0.21072857081890106, 0.03228483349084854, 0.22392968833446503, 0.130387082695961, 0.5947916507720947, 0.8742749094963074, 0.8987970948219299, 0.7172414064407349, 0.9023902416229248]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 19461, 7513, 85466, 17672, 56769, 6633, 220, 37088, 32488], "first_top_logits": [26.25, 21.625, 21.5, 21.25, 21.25, 20.375, 19.375, 19.25, 18.75, 18.25], "source_correct": false, "target_correct": false, "seconds": 0.4568949160166085, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4012589056}}
{"run_id": "eval-22/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -7.0929179191589355, "target_logp": -48.821746826171875, "log_odds": -41.72882890701294, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-6.501625061035156, -0.5910685062408447, -0.00022480344341602176]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-47.001625061035156, -1.8189048767089844, -0.0012180536286905408]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 635.0573120117188, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.2429809570312]], "actual_norm": [[237.26779174804688, 0.12294193357229233, 16.219467163085938, 181.9257049560547, 137.43594360351562, 228.81753540039062, 30.143882751464844, 226.9967041015625, 126.51493835449219, 635.1617431640625, 1124.9891357421875, 862.4203491210938, 781.36376953125, 935.1024169921875]], "relative_norm": [[0.015001848340034485, 0.00015477239503525198, 0.014033238403499126, 0.16025228798389435, 0.1294039636850357, 0.21164803206920624, 0.029030537232756615, 0.2288316786289215, 0.1319005936384201, 0.6025543808937073, 0.8695545196533203, 0.9053424596786499, 0.7127854228019714, 0.9033824801445007]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.88345336914062, 137.37698364257812, 228.79539489746094, 30.158061981201172, 227.02777099609375, 126.50897216796875, 635.0573120117188, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.2429809570312]], "actual_norm": [[237.26779174804688, 0.12294193357229233, 16.219467163085938, 181.9257049560547, 137.43594360351562, 228.81753540039062, 30.143882751464844, 226.9967041015625, 126.51493835449219, 635.1617431640625, 1124.9891357421875, 862.4203491210938, 781.36376953125, 935.1024169921875]], "relative_norm": [[0.015001848340034485, 0.00015477239503525198, 0.014033238403499126, 0.16025228798389435, 0.1294039636850357, 0.21164803206920624, 0.029030537232756615, 0.2288316786289215, 0.1319005936384201, 0.6025543808937073, 0.8695545196533203, 0.9053424596786499, 0.7127854228019714, 0.9033824801445007]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungary forint.  What else do we know about Sz", "output_ids": [42776, 369, 396, 13, 220, 3555, 770, 653, 582, 1414, 911, 44190], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.3301239013672, 33.43550109863281, 222.82388305664062, 125.29711151123047, 625.5267333984375, 1130.1561279296875, 854.1030883789062, 784.0282592773438, 933.7200317382812]], "actual_norm": [[237.26779174804688, 0.0520724318921566, 20.986572265625, 180.92987060546875, 136.83164978027344, 227.33685302734375, 33.4898567199707, 222.83822631835938, 125.26361846923828, 625.5509033203125, 1130.0880126953125, 854.0621948242188, 784.0132446289062, 933.639892578125]], "relative_norm": [[0.015001848340034485, 6.653050513705239e-05, 0.018165895715355873, 0.15984810888767242, 0.12867312133312225, 0.21066758036613464, 0.03233252093195915, 0.22395749390125275, 0.1303279846906662, 0.5948036909103394, 0.8742990493774414, 0.8988494873046875, 0.7172882556915283, 0.9022774696350098]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [42776, 56769, 103670, 89375, 25523, 69595, 472, 18295, 386, 5429], "first_top_logits": [29.625, 23.125, 20.375, 18.875, 16.375, 15.9375, 15.8125, 15.625, 13.1875, 13.125], "source_correct": true, "target_correct": false, "seconds": 0.42556762509047985, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-22/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.09044885635376, "target_logp": -17.891332626342773, "log_odds": -13.800883769989014, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-3.3862123489379883, -0.70405113697052, -0.0001854724541772157]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-16.011213302612305, -1.8778611421585083, -0.0022590849548578262]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.158061981201172, 227.0277557373047, 126.50896453857422, 635.0572509765625, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.242919921875]], "actual_norm": [[236.99099731445312, 0.12726791203022003, 16.17294692993164, 181.87330627441406, 137.4217529296875, 228.81961059570312, 30.24599838256836, 226.97975158691406, 126.50064086914062, 635.0123291015625, 1125.205810546875, 862.4719848632812, 781.3654174804688, 935.2537841796875]], "relative_norm": [[0.01498434692621231, 0.00016021839110180736, 0.013992988504469395, 0.1602061241865158, 0.12939059734344482, 0.21164995431900024, 0.02912888117134571, 0.22881458699703217, 0.13188569247722626, 0.6024126410484314, 0.8697220087051392, 0.9053966403007507, 0.7127869129180908, 0.9035286903381348]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.158061981201172, 227.0277557373047, 126.50896453857422, 635.0572509765625, 1125.1011962890625, 862.4371948242188, 781.3258056640625, 935.242919921875]], "actual_norm": [[236.99099731445312, 0.12726791203022003, 16.17294692993164, 181.87330627441406, 137.4217529296875, 228.81961059570312, 30.24599838256836, 226.97975158691406, 126.50064086914062, 635.0123291015625, 1125.205810546875, 862.4719848632812, 781.3654174804688, 935.2537841796875]], "relative_norm": [[0.01498434692621231, 0.00016021839110180736, 0.013992988504469395, 0.1602061241865158, 0.12939059734344482, 0.21164995431900024, 0.02912888117134571, 0.22881458699703217, 0.13188569247722626, 0.6024126410484314, 0.8697220087051392, 0.9053966403007507, 0.7127869129180908, 0.9035286903381348]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  What is the currency used in the country containing", "output_ids": [17672, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.244384765625, 0.1332213282585144, 20.935914993286133, 180.9315948486328, 136.80807495117188, 227.33010864257812, 33.43550109863281, 222.82386779785156, 125.29711151123047, 625.5267333984375, 1130.1561279296875, 854.10302734375, 784.0281982421875, 933.719970703125]], "actual_norm": [[236.99099731445312, 0.0524776317179203, 20.942527770996094, 180.89088439941406, 136.83682250976562, 227.3363800048828, 33.53871154785156, 222.7841033935547, 125.32426452636719, 625.5308837890625, 1130.132568359375, 854.0299072265625, 783.9890747070312, 933.6844482421875]], "relative_norm": [[0.01498434692621231, 6.704821134917438e-05, 0.018127771094441414, 0.15981365740299225, 0.12867799401283264, 0.21066713333129883, 0.03237968683242798, 0.2239031046628952, 0.13039109110832214, 0.594784677028656, 0.8743335604667664, 0.8988155126571655, 0.7172661423683167, 0.9023205637931824]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 56769, 7513, 472, 468, 85466, 7855, 32488, 37088], "first_top_logits": [24.875, 24.5, 21.875, 20.125, 20.0, 19.875, 19.375, 18.625, 18.5, 18.375], "source_correct": false, "target_correct": false, "seconds": 0.4343754998408258, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-22/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.119993209838867, "target_logp": -17.70001220703125, "log_odds": -14.580018997192383, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-2.577728271484375, -0.5420480966567993, -0.00021705655672121793]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-15.546478271484375, -2.151975631713867, -0.0015589953400194645]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.8834686279297, 137.37698364257812, 228.79539489746094, 30.158063888549805, 227.02777099609375, 126.50897979736328, 635.0573120117188, 1125.1011962890625, 862.437255859375, 781.3258666992188, 935.2430419921875]], "actual_norm": [[237.1091766357422, 0.13315518200397491, 16.20315933227539, 181.87828063964844, 137.41062927246094, 228.7781524658203, 30.143505096435547, 226.99298095703125, 126.49353790283203, 635.0037231445312, 1124.9775390625, 862.5595092773438, 781.3868408203125, 935.1854858398438]], "relative_norm": [[0.01499181892722845, 0.00016762991435825825, 0.014019128866493702, 0.160210520029068, 0.12938012182712555, 0.21161159873008728, 0.029030174016952515, 0.22882792353630066, 0.13187828660011292, 0.6024044752120972, 0.869545578956604, 0.9054885506629944, 0.7128064632415771, 0.9034627079963684]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190892815589905, 16.1784725189209, 181.8834686279297, 137.37698364257812, 228.79539489746094, 30.158063888549805, 227.02777099609375, 126.50897979736328, 635.0573120117188, 1125.1011962890625, 862.437255859375, 781.3258666992188, 935.2430419921875]], "actual_norm": [[237.1091766357422, 0.13315518200397491, 16.20315933227539, 181.87828063964844, 137.41062927246094, 228.7781524658203, 30.143505096435547, 226.99298095703125, 126.49353790283203, 635.0037231445312, 1124.9775390625, 862.5595092773438, 781.3868408203125, 935.1854858398438]], "relative_norm": [[0.01499181892722845, 0.00016762991435825825, 0.014019128866493702, 0.160210520029068, 0.12938012182712555, 0.21161159873008728, 0.029030174016952515, 0.22882792353630066, 0.13187828660011292, 0.6024044752120972, 0.869545578956604, 0.9054885506629944, 0.7128064632415771, 0.9034627079963684]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  What is the currency used in the country containing", "output_ids": [17672, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213431596756, 20.935916900634766, 180.93161010742188, 136.80810546875, 227.33013916015625, 33.43550491333008, 222.82388305664062, 125.297119140625, 625.5267333984375, 1130.1561279296875, 854.1030883789062, 784.0283203125, 933.7200317382812]], "actual_norm": [[237.1091766357422, 0.056986004114151, 20.9071102142334, 180.94696044921875, 136.82696533203125, 227.28517150878906, 33.41572189331055, 222.85386657714844, 125.28910827636719, 625.542236328125, 1130.208251953125, 854.1989135742188, 784.09521484375, 933.8848876953125]], "relative_norm": [[0.01499181892722845, 7.280834688572213e-05, 0.01809711381793022, 0.1598632037639618, 0.12866872549057007, 0.21061968803405762, 0.0322609469294548, 0.22397321462631226, 0.13035450875759125, 0.5947954654693604, 0.8743920922279358, 0.8989933729171753, 0.7173632383346558, 0.9025142788887024]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 56769, 37088, 369, 13, 547, 3639, 10038, 44190], "first_top_logits": [20.25, 19.875, 18.5, 18.375, 18.0, 17.75, 17.25, 17.125, 17.125, 16.875], "source_correct": false, "target_correct": false, "seconds": 0.4371015829965472, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-22/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-22", "split": "eval", "country": "Hungary", "currency": "Hungarian forint", "aliases": ["forint", "HUF"], "city": "Szeged", "target_country": "Israel", "target_currency": "Israeli shekel", "target_aliases": ["shekel", "ILS"], "prompt": "Fact: The currency used in the country containing Szeged is the", "source_continuation": " Hungarian forint", "target_continuation": " Israeli shekel"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.6195553541183472, "target_logp": -14.191165924072266, "log_odds": -13.571610569953918, "token_scores": {"source": {"ids": [56769, 369, 396], "log_probs": [-0.26279309391975403, -0.356519877910614, -0.00024232311989180744]}, "target": {"ids": [15907, 1340, 17659], "log_probs": [-12.70029354095459, -1.4893214702606201, -0.001551139634102583]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.15806007385254, 227.02777099609375, 126.50896453857422, 635.0572509765625, 1125.10107421875, 862.4371337890625, 781.3257446289062, 935.242919921875]], "actual_norm": [[236.99652099609375, 0.13541673123836517, 16.213977813720703, 181.8157958984375, 137.3481903076172, 228.75857543945312, 30.200042724609375, 226.99557495117188, 126.52033996582031, 635.013671875, 1125.2476806640625, 862.48681640625, 781.35009765625, 935.1836547851562]], "relative_norm": [[0.014984696172177792, 0.00017047699657268822, 0.014028489589691162, 0.16015547513961792, 0.12932133674621582, 0.21159349381923676, 0.029084622859954834, 0.22883053123950958, 0.13190622627735138, 0.6024138927459717, 0.8697543740272522, 0.9054121971130371, 0.7127729654312134, 0.9034609198570251]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.25190889835357666, 16.1784725189209, 181.88343811035156, 137.37698364257812, 228.79537963867188, 30.15806007385254, 227.02777099609375, 126.50896453857422, 635.0572509765625, 1125.10107421875, 862.4371337890625, 781.3257446289062, 935.242919921875]], "actual_norm": [[236.99652099609375, 0.13541673123836517, 16.213977813720703, 181.8157958984375, 137.3481903076172, 228.75857543945312, 30.200042724609375, 226.99557495117188, 126.52033996582031, 635.013671875, 1125.2476806640625, 862.48681640625, 781.35009765625, 935.1836547851562]], "relative_norm": [[0.014984696172177792, 0.00017047699657268822, 0.014028489589691162, 0.16015547513961792, 0.12932133674621582, 0.21159349381923676, 0.029084622859954834, 0.22883053123950958, 0.13190622627735138, 0.6024138927459717, 0.8697543740272522, 0.9054121971130371, 0.7127729654312134, 0.9034609198570251]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Hungarian forint.\n\nAnother fact: The currency used in the", "output_ids": [56769, 369, 396, 382, 14037, 2097, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 44190, 791, 291, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[237.24440002441406, 0.1332213282585144, 20.935916900634766, 180.93161010742188, 136.80809020996094, 227.33010864257812, 33.43550109863281, 222.82386779785156, 125.29711151123047, 625.5267333984375, 1130.1561279296875, 854.10302734375, 784.0281982421875, 933.719970703125]], "actual_norm": [[236.99652099609375, 0.05747038871049881, 20.956981658935547, 180.93589782714844, 136.800537109375, 227.3764190673828, 33.460350036621094, 222.86021423339844, 125.29371643066406, 625.4559326171875, 1130.181396484375, 854.1117553710938, 784.0568237304688, 933.7643432617188]], "relative_norm": [[0.014984696172177792, 7.342721801251173e-05, 0.018140282481908798, 0.15985342860221863, 0.12864387035369873, 0.2107042372226715, 0.032304033637046814, 0.22397959232330322, 0.13035930693149567, 0.5947133898735046, 0.874371349811554, 0.898901641368866, 0.7173281311988831, 0.9023977518081665]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [56769, 17672, 19461, 3639, 444, 220, 42364, 11888, 17692, 19551], "first_top_logits": [22.625, 20.875, 20.5, 19.125, 18.0, 17.375, 16.75, 16.625, 16.375, 16.25], "source_correct": true, "target_correct": false, "seconds": 0.4177326252683997, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/readout", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 4232.5, "target_rank": 7624.0, "source_logit": 0.91015625, "target_logit": -0.0439453125, "top_ids": [15973, 11413, 8330, 38352, 45418, 28453, 6364, 49029, 31491, 7513, 4505, 7065], "top_tokens": ["currency", " currency", " Australia", "uenta", ".currency", " Currency", " English", "ctica", "uesto", " European", " Europe", "ican"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 1523.0, "target_rank": 11952.0, "source_logit": 2.734375, "target_logit": -0.478515625, "top_ids": [11413, 15973, 28453, 96694, 3001, 53661, 26321, 45418, 7065, 5088, 3693, 42189], "top_tokens": [" currency", "currency", " Currency", "onis", "ica", "cla", "Currency", ".currency", "ican", "urrency", " American", "etz"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 347.5, "target_rank": 546.0, "source_logit": 15.625, "target_logit": 14.5625, "top_ids": [11413, 35602, 32488, 7513, 8330, 90475, 22956, 6489, 36445, 45418, 7214, 85466], "top_tokens": [" currency", " currencies", " euros", " European", " Australia", "(currency", "\".\n", " international", " internationally", ".currency", " foreign", " Euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 106.0, "target_rank": 751.5, "source_logit": 15.6875, "target_logit": 12.375, "top_ids": [11413, 35602, 7214, 13224, 7855, 47412, 3220, 65001, 78253, 21130, 34710, 15973], "top_tokens": [" currency", " currencies", " foreign", " Britain", " British", " foreigners", " money", "\"\u2014", " treasury", " parliament", "_currency", "currency"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 3144.0, "target_rank": 13141.0, "source_logit": 1.171875, "target_logit": -1.078125, "top_ids": [21251, 28453, 3776, 603, 7748, 3731, 12751, 55964, 4229, 9127, 11413, 5737], "top_tokens": ["onia", " Currency", " One", "one", " Indian", " Red", " Roman", "\u2011", " Ge", " Rock", " currency", " Group"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 2813.0, "target_rank": 7575.0, "source_logit": 1.0859375, "target_logit": -0.408203125, "top_ids": [11413, 7179, 364, 11766, 479, 21251, 18174, 2932, 28453, 6489, 64, 356], "top_tokens": [" currency", " International", " '", " Standard", " G", "onia", "oret", " She", " Currency", " international", "a", " C"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1759.0, "target_rank": 3136.5, "source_logit": 12.875, "target_logit": 11.9375, "top_ids": [11413, 35602, 3146, 5837, 5313, 13867, 28453, 32488, 30176, 7513, 43895, 7214], "top_tokens": [" currency", " currencies", " country", " countries", " national", " county", " Currency", " euros", " counties", " European", " tariffs", " foreign"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 400.0, "target_rank": 1449.5, "source_logit": 14.4375, "target_logit": 12.125, "top_ids": [11413, 35602, 7513, 7855, 17672, 37198, 6489, 7214, 5837, 105492, 103408, 94936], "top_tokens": [" currency", " currencies", " European", " British", " euro", " Commonwealth", " international", " foreign", " countries", "\u4e2d\u534e\u4eba\u6c11", "\u6b27\u5143", " denomination"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3293.5, "target_rank": 9139.0, "source_logit": 0.765625, "target_logit": -0.90625, "top_ids": [3693, 4787, 279, 89478, 4229, 5434, 4102, 21251, 6364, 12022, 2240, 1083], "top_tokens": [" American", " North", " the", "\u2026\n", " Ge", "\u2026\n\n", "\u00a0", "onia", " English", " \u2026\n\n", " None", " also"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 304.5, "target_rank": 7471.5, "source_logit": 3.984375, "target_logit": -0.72265625, "top_ids": [1852, 5938, 3693, 4787, 4102, 271, 4882, 3639, 479, 362, 547, 7179], "top_tokens": [" same", " German", " American", " North", "\u00a0", "\n\n", " South", " United", " G", " A", " U", " International"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 11756.5, "target_rank": 9200.0, "source_logit": 6.1875, "target_logit": 6.78125, "top_ids": [11413, 35602, 103408, 104142, 28453, 113883, 32488, 90475, 26321, 106333, 115160, 112455], "top_tokens": [" currency", " currencies", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " Currency", "\u672c\u56fd", " euros", "(currency", "Currency", "\u5916\u6c47", "\u5176\u4ed6\u56fd\u5bb6", "\u82f1\u9551"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 4132.5, "target_rank": 12739.0, "source_logit": 3.984375, "target_logit": 1.4375, "top_ids": [104142, 103408, 106333, 116866, 109442, 101237, 110065, 113883, 106127, 112397, 105579, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u5916\u6c47", "\u76f4\u8f96\u5e02", "\u8d22\u653f\u90e8", "\u7f8e\u5143", "\u6c47\u7387", "\u672c\u56fd", "\u6b27\u76df", "\u56fd\u5bb6\u6807\u51c6", "\u6fb3\u5927\u5229\u4e9a", "\u6210\u5458\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 66.0, "target_rank": 2390.5, "source_logit": 12.125, "target_logit": 3.40625, "top_ids": [11413, 11192, 104142, 28453, 17692, 112455, 35602, 80189, 32488, 103408, 19551, 85466], "top_tokens": [" currency", " dollars", "\u4eba\u6c11\u5e01", " Currency", " dollar", "\u82f1\u9551", " currencies", " Dollars", " euros", "\u6b27\u5143", " USD", " Euros"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 11.0, "target_rank": 1431.5, "source_logit": 18.5, "target_logit": 3.84375, "top_ids": [11413, 15907, 104142, 17692, 28453, 11192, 36266, 3639, 35602, 19551, 6778, 17672], "top_tokens": [" currency", " Israeli", "\u4eba\u6c11\u5e01", " dollar", " Currency", " dollars", " Hebrew", " United", " currencies", " USD", " Israel", " euro"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 122.0, "target_rank": 1916.0, "source_logit": 18.875, "target_logit": 9.5, "top_ids": [11413, 104142, 35602, 11192, 112455, 28453, 103408, 80189, 32488, 17692, 101237, 101406], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " dollars", "\u82f1\u9551", " Currency", "\u6b27\u5143", " Dollars", " euros", " dollar", "\u7f8e\u5143", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 42.5, "target_rank": 1654.0, "source_logit": 20.125, "target_logit": 7.0625, "top_ids": [104142, 35602, 11413, 28453, 101237, 103408, 112455, 101406, 17692, 110065, 34710, 11192], "top_tokens": ["\u4eba\u6c11\u5e01", " currencies", " currency", " Currency", "\u7f8e\u5143", "\u6b27\u5143", "\u82f1\u9551", "\u8d27\u5e01", " dollar", "\u6c47\u7387", "_currency", " dollars"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 20.0, "target_rank": 4530.0, "source_logit": 18.625, "target_logit": 7.0, "top_ids": [279, 7855, 2598, 15907, 3881, 1483, 19461, 3639, 36266, 17089, 537, 78252], "top_tokens": [" the", " British", " called", " Israeli", " known", " used", " Euro", " United", " Hebrew", " Jordan", " not", " Pound"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 6.0, "target_rank": 6350.0, "source_logit": 22.0, "target_logit": 4.6875, "top_ids": [15907, 36266, 1340, 7855, 2932, 6778, 1532, 32489, 17089, 21841, 3639, 19461], "top_tokens": [" Israeli", " Hebrew", " she", " British", " She", " Israel", " New", " Egyptian", " Jordan", " Palestinian", " United", " Euro"]}], "source_token": 6778, "target_token": 23190, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Ha", "ifa", " is", " the"], "seconds": 1.165655457880348}
{"run_id": "eval-23/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0766308307647705, "target_logp": -14.10993480682373, "log_odds": -13.03330397605896, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.018440593034029007, -1.0544103384017944, -0.0037798879202455282]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-13.830940246582031, -0.2781941294670105, -0.0007996937492862344]}}, "score_trace": {"source": [], "target": []}, "output": " Israeli shekel.\n\nFact: The currency used in the country", "output_ids": [15907, 1340, 17659, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [], "first_top_ids": [15907, 36266, 1340, 7855, 2932, 6778, 1532, 32489, 17089, 3639], "first_top_logits": [28.625, 23.0, 22.75, 22.625, 22.625, 22.0, 21.5, 21.375, 21.125, 20.375], "source_correct": true, "target_correct": false, "seconds": 0.4595324592664838, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0766308307647705, "target_logp": -14.10993480682373, "log_odds": -13.03330397605896, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.018440593034029007, -1.0544103384017944, -0.0037798879202455282]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-13.830940246582031, -0.2781941294670105, -0.0007996937492862344]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli shekel.\n\nFact: The currency used in the country", "output_ids": [15907, 1340, 17659, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 36266, 1340, 7855, 2932, 6778, 1532, 32489, 17089, 3639], "first_top_logits": [28.625, 23.0, 22.75, 22.625, 22.625, 22.0, 21.5, 21.375, 21.125, 20.375], "source_correct": true, "target_correct": false, "seconds": 0.4525401252321899, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -27.953886032104492, "target_logp": -0.32129618525505066, "log_odds": 27.63258984684944, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-26.747055053710938, -1.2025858163833618, -0.0042463853023946285]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-0.16111794114112854, -0.15951840579509735, -0.0006598440813831985]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390563964844, 654.2545776367188]], "actual_norm": [[96.88777923583984, 52.93351364135742, 12.358449935913086, 138.8209228515625, 90.29010772705078, 179.620361328125, 92.13438415527344, 60.251670837402344, 31.98891258239746, 131.26840209960938, 391.8310241699219, 498.4093933105469, 654.1677856445312]], "relative_norm": [[0.006125971674919128, 0.06663834303617477, 0.010692649520933628, 0.1222827211022377, 0.08501340448856354, 0.16614240407943726, 0.08873146027326584, 0.06073872745037079, 0.03335065767168999, 0.12159591913223267, 0.43165674805641174, 0.4477425217628479, 0.6284487247467041]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390563964844, 654.2545776367188]], "actual_norm": [[96.88777923583984, 52.93351364135742, 12.358449935913086, 138.8209228515625, 90.29010772705078, 179.620361328125, 92.13438415527344, 60.251670837402344, 31.98891258239746, 131.26840209960938, 391.8310241699219, 498.4093933105469, 654.1677856445312]], "relative_norm": [[0.006125971674919128, 0.06663834303617477, 0.010692649520933628, 0.1222827211022377, 0.08501340448856354, 0.16614240407943726, 0.08873146027326584, 0.06073872745037079, 0.03335065767168999, 0.12159591913223267, 0.43165674805641174, 0.4477425217628479, 0.6284487247467041]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.\n\nFact: The currency used in the country", "output_ids": [30109, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.148619651794434, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539419174194336, 129.43148803710938, 389.2738342285156, 483.4374084472656, 665.1746215820312]], "actual_norm": [[96.88777923583984, 51.86480712890625, 14.185067176818848, 137.3540496826172, 87.75621032714844, 175.22848510742188, 96.29241943359375, 60.45827102661133, 29.59929084777832, 129.4054718017578, 389.2456970214844, 483.49627685546875, 665.2108154296875]], "relative_norm": [[0.006125971674919128, 0.06626523286104202, 0.012278539128601551, 0.12134969234466553, 0.08252378553152084, 0.16238000988960266, 0.09296476095914841, 0.060761939734220505, 0.030795980244874954, 0.12019756436347961, 0.42888662219047546, 0.4347611963748932, 0.6416807174682617]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 23190, 43680, 5052, 84574, 110055, 19461, 4492, 17672, 37088], "first_top_logits": [28.25, 26.5, 23.0, 23.0, 22.5, 22.125, 21.625, 20.75, 20.625, 20.5], "source_correct": false, "target_correct": true, "seconds": 0.42070008302107453, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.0370888710021973, "target_logp": -14.928780555725098, "log_odds": -12.8916916847229, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.5139175653457642, -1.5193899869918823, -0.0037813130766153336]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-14.701417922973633, -0.2268027514219284, -0.0005594118847511709]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.29171371459961, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.1296615600586, 60.29090881347656, 31.99578094482422, 131.2808380126953, 391.8334655761719, 498.43902587890625, 654.2545776367188]], "actual_norm": [[96.88272094726562, 53.01782989501953, 12.327152252197266, 138.79266357421875, 90.29666137695312, 179.614501953125, 92.15062713623047, 60.316429138183594, 32.04558563232422, 131.31610107421875, 391.8387451171875, 498.4522705078125, 654.1966552734375]], "relative_norm": [[0.006125651765614748, 0.06674448400735855, 0.010665570385754108, 0.12225782871246338, 0.08501957356929779, 0.1661369800567627, 0.08874709904193878, 0.060804009437561035, 0.03340974450111389, 0.12164010107517242, 0.4316652715206146, 0.4477810263633728, 0.6284764409065247]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.29171371459961, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.1296615600586, 60.29090881347656, 31.99578094482422, 131.2808380126953, 391.8334655761719, 498.43902587890625, 654.2545776367188]], "actual_norm": [[96.88272094726562, 53.01782989501953, 12.327152252197266, 138.79266357421875, 90.29666137695312, 179.614501953125, 92.15062713623047, 60.316429138183594, 32.04558563232422, 131.31610107421875, 391.8387451171875, 498.4522705078125, 654.1966552734375]], "relative_norm": [[0.006125651765614748, 0.06674448400735855, 0.010665570385754108, 0.12225782871246338, 0.08501957356929779, 0.1661369800567627, 0.08874709904193878, 0.060804009437561035, 0.03340974450111389, 0.12164010107517242, 0.4316652715206146, 0.4477810263633728, 0.6284764409065247]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli Shekel.\n\nFact: The currency used in the country", "output_ids": [15907, 2932, 17659, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.14862060546875, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539417266845703, 129.4314727783203, 389.2738342285156, 483.4374084472656, 665.1746215820312]], "actual_norm": [[96.88272094726562, 51.881771087646484, 14.216714859008789, 137.41973876953125, 87.73492431640625, 175.16087341308594, 96.29576110839844, 60.52335739135742, 29.563894271850586, 129.43069458007812, 389.2646789550781, 483.54754638671875, 665.156005859375]], "relative_norm": [[0.006125651765614748, 0.06628690659999847, 0.01230593305081129, 0.12140773236751556, 0.0825037732720375, 0.16231735050678253, 0.09296798706054688, 0.0608273521065712, 0.03075915388762951, 0.12022098898887634, 0.42890751361846924, 0.43480730056762695, 0.641627848148346]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 1965, 6778, 36266, 17692, 2274, 3639, 1140, 21841, 6633], "first_top_logits": [26.375, 25.875, 22.75, 21.625, 20.875, 20.25, 20.125, 19.625, 19.5, 19.375], "source_correct": true, "target_correct": false, "seconds": 0.4264346668496728, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1360666751861572, "target_logp": -37.655887603759766, "log_odds": -35.51982092857361, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.7717408537864685, -1.3613454103469849, -0.002980435499921441]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-37.2717399597168, -0.3834458589553833, -0.0007033258443698287]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390563964844, 654.2545776367188]], "actual_norm": [[96.90357208251953, 52.93779373168945, 12.352404594421387, 138.827392578125, 90.2739486694336, 179.63320922851562, 92.09620666503906, 60.278045654296875, 31.991329193115234, 131.243408203125, 391.8910827636719, 498.4317626953125, 654.2282104492188]], "relative_norm": [[0.006126970052719116, 0.06664372980594635, 0.010687419213354588, 0.12228842079639435, 0.08499819040298462, 0.16615428030490875, 0.08869469165802002, 0.06076531484723091, 0.033353179693222046, 0.12157276272773743, 0.4317229390144348, 0.4477626085281372, 0.6285067796707153]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390563964844, 654.2545776367188]], "actual_norm": [[96.90357208251953, 52.93779373168945, 12.352404594421387, 138.827392578125, 90.2739486694336, 179.63320922851562, 92.09620666503906, 60.278045654296875, 31.991329193115234, 131.243408203125, 391.8910827636719, 498.4317626953125, 654.2282104492188]], "relative_norm": [[0.006126970052719116, 0.06664372980594635, 0.010687419213354588, 0.12228842079639435, 0.08499819040298462, 0.16615428030490875, 0.08869469165802002, 0.06076531484723091, 0.033353179693222046, 0.12157276272773743, 0.4317229390144348, 0.4477626085281372, 0.6285067796707153]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israel New Shekel.\n\nAnother fact: The currency used in", "output_ids": [6778, 1532, 2932, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.148619651794434, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539419174194336, 129.43148803710938, 389.2738342285156, 483.4374084472656, 665.1746215820312]], "actual_norm": [[96.90357208251953, 51.88034439086914, 14.18665599822998, 137.36122131347656, 87.76414489746094, 175.18882751464844, 96.31395721435547, 60.506248474121094, 29.63409423828125, 129.45211791992188, 389.330810546875, 483.3677673339844, 665.2158203125]], "relative_norm": [[0.006126970052719116, 0.06628508120775223, 0.012279914692044258, 0.12135603278875351, 0.08253125101327896, 0.16234326362609863, 0.09298555552959442, 0.060810159891843796, 0.030832191929221153, 0.12024088948965073, 0.42898038029670715, 0.4346456527709961, 0.6416855454444885]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6778, 15907, 2932, 36266, 1340, 501, 13350, 108781, 98124, 17692], "first_top_logits": [26.25, 26.125, 22.0, 21.625, 20.75, 19.5, 17.25, 17.25, 16.625, 16.375], "source_correct": true, "target_correct": false, "seconds": 0.44976770924404263, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.3568034172058105, "target_logp": -12.166370391845703, "log_odds": -10.809566974639893, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.12772005796432495, -1.2235995531082153, -0.0054838224314153194]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-11.75271987915039, -0.4129289984703064, -0.0007219092221930623]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.63893127441406, 92.1296615600586, 60.2909049987793, 31.995777130126953, 131.28082275390625, 391.8334655761719, 498.43902587890625, 654.2545166015625]], "actual_norm": [[96.83702087402344, 52.98702621459961, 12.302899360656738, 138.7931365966797, 90.36840057373047, 179.61146545410156, 92.21683502197266, 60.272708892822266, 31.98310089111328, 131.2959747314453, 391.8650817871094, 498.4263000488281, 654.2274780273438]], "relative_norm": [[0.0061227623373270035, 0.06670571118593216, 0.010644586756825447, 0.1222582459449768, 0.0850871205329895, 0.16613417863845825, 0.08881086111068726, 0.06075993552803993, 0.03334460034966469, 0.12162145972251892, 0.43169426918029785, 0.44775769114494324, 0.628506064414978]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.63893127441406, 92.1296615600586, 60.2909049987793, 31.995777130126953, 131.28082275390625, 391.8334655761719, 498.43902587890625, 654.2545166015625]], "actual_norm": [[96.83702087402344, 52.98702621459961, 12.302899360656738, 138.7931365966797, 90.36840057373047, 179.61146545410156, 92.21683502197266, 60.272708892822266, 31.98310089111328, 131.2959747314453, 391.8650817871094, 498.4263000488281, 654.2274780273438]], "relative_norm": [[0.0061227623373270035, 0.06670571118593216, 0.010644586756825447, 0.1222582459449768, 0.0850871205329895, 0.16613417863845825, 0.08881086111068726, 0.06075993552803993, 0.03334460034966469, 0.12162145972251892, 0.43169426918029785, 0.44775769114494324, 0.628506064414978]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli shekel.\n\nFact: The currency used in the country", "output_ids": [15907, 1340, 17659, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 51.85837936401367, 14.148618698120117, 137.3745880126953, 87.7684555053711, 175.21165466308594, 96.29875183105469, 60.48415756225586, 29.539417266845703, 129.4314727783203, 389.2738037109375, 483.4373779296875, 665.1746215820312]], "actual_norm": [[96.83702087402344, 51.840091705322266, 14.142302513122559, 137.46112060546875, 87.87090301513672, 175.23001098632812, 96.34798431396484, 60.480533599853516, 29.524280548095703, 129.50384521484375, 389.2757568359375, 483.490478515625, 665.1312866210938]], "relative_norm": [[0.0061227623373270035, 0.06623365730047226, 0.01224152185022831, 0.12144429236650467, 0.08263164013624191, 0.16238142549991608, 0.09301841259002686, 0.06078431382775307, 0.030717937275767326, 0.12028893828392029, 0.4289197325706482, 0.43475598096847534, 0.6416040062904358]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 7855, 19461, 2932, 3639, 17672, 1340, 23734, 36266, 32489], "first_top_logits": [26.0, 23.5, 21.875, 21.25, 21.0, 20.375, 19.875, 19.75, 19.5, 19.5], "source_correct": true, "target_correct": false, "seconds": 0.4157971669919789, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.7289553880691528, "target_logp": -11.6336030960083, "log_odds": -9.904647707939148, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.32312092185020447, -1.4015525579452515, -0.004281877540051937]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-11.448121070861816, -0.18483296036720276, -0.0006493605324067175]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63894653320312, 92.12967681884766, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390869140625, 654.2545776367188]], "actual_norm": [[96.91927337646484, 52.99755096435547, 12.352665901184082, 138.75584411621094, 90.33192443847656, 179.64727783203125, 92.12714385986328, 60.311954498291016, 31.992265701293945, 131.30316162109375, 391.81256103515625, 498.4680480957031, 654.163330078125]], "relative_norm": [[0.006127962842583656, 0.06671895831823349, 0.010687645524740219, 0.12222539633512497, 0.08505277335643768, 0.16616730391979218, 0.08872448652982712, 0.0607994981110096, 0.03335415571928024, 0.12162811309099197, 0.43163642287254333, 0.44779521226882935, 0.6284444332122803]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63894653320312, 92.12967681884766, 60.29090881347656, 31.995779037475586, 131.2808380126953, 391.83349609375, 498.4390869140625, 654.2545776367188]], "actual_norm": [[96.91927337646484, 52.99755096435547, 12.352665901184082, 138.75584411621094, 90.33192443847656, 179.64727783203125, 92.12714385986328, 60.311954498291016, 31.992265701293945, 131.30316162109375, 391.81256103515625, 498.4680480957031, 654.163330078125]], "relative_norm": [[0.006127962842583656, 0.06671895831823349, 0.010687645524740219, 0.12222539633512497, 0.08505277335643768, 0.16616730391979218, 0.08872448652982712, 0.0607994981110096, 0.03335415571928024, 0.12162811309099197, 0.43163642287254333, 0.44779521226882935, 0.6284444332122803]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli new shekel.\n\nAnother fact: The currency used in", "output_ids": [15907, 501, 1340, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 51.85838317871094, 14.14862060546875, 137.3745880126953, 87.7684555053711, 175.21168518066406, 96.29875183105469, 60.484161376953125, 29.539419174194336, 129.43148803710938, 389.2738342285156, 483.43743896484375, 665.1746215820312]], "actual_norm": [[96.91927337646484, 51.87762451171875, 14.202552795410156, 137.39434814453125, 87.7420425415039, 175.18060302734375, 96.27130126953125, 60.55527877807617, 29.561262130737305, 129.41946411132812, 389.2223205566406, 483.484130859375, 665.2025756835938]], "relative_norm": [[0.006127962842583656, 0.06628160923719406, 0.012293674983084202, 0.12138529866933823, 0.08251046389341354, 0.16233563423156738, 0.09294437617063522, 0.06085943430662155, 0.03075641393661499, 0.12021055817604065, 0.4288608431816101, 0.4347502887248993, 0.6416727900505066]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 19461, 17672, 1532, 17692, 36266, 6778, 7855, 3639, 21841], "first_top_logits": [25.0, 22.75, 22.0, 21.875, 20.75, 20.5, 20.125, 20.0, 19.875, 19.5], "source_correct": true, "target_correct": false, "seconds": 0.4301872090436518, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-23/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-23", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Haifa", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Haifa is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.028314471244812, "target_logp": -13.982157707214355, "log_odds": -12.953843235969543, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.02124960348010063, -1.0022399425506592, -0.004824900534003973]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-13.708749771118164, -0.2728445529937744, -0.0005638201837427914]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.638916015625, 92.1296615600586, 60.2909049987793, 31.995777130126953, 131.28082275390625, 391.8334655761719, 498.43902587890625, 654.2545776367188]], "actual_norm": [[96.7635498046875, 52.94948196411133, 12.33750057220459, 138.76223754882812, 90.23460388183594, 179.59701538085938, 92.14163208007812, 60.30009841918945, 32.0026741027832, 131.21807861328125, 391.8042297363281, 498.44207763671875, 654.19873046875]], "relative_norm": [[0.0061181169003248215, 0.06665844470262527, 0.010674524120986462, 0.12223102897405624, 0.08496114611625671, 0.16612081229686737, 0.08873844146728516, 0.060787543654441833, 0.03336500748991966, 0.12154930084943771, 0.4316272437572479, 0.4477718770503998, 0.628478467464447]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.638916015625, 92.1296615600586, 60.2909049987793, 31.995777130126953, 131.28082275390625, 391.8334655761719, 498.43902587890625, 654.2545776367188]], "actual_norm": [[96.7635498046875, 52.94948196411133, 12.33750057220459, 138.76223754882812, 90.23460388183594, 179.59701538085938, 92.14163208007812, 60.30009841918945, 32.0026741027832, 131.21807861328125, 391.8042297363281, 498.44207763671875, 654.19873046875]], "relative_norm": [[0.0061181169003248215, 0.06665844470262527, 0.010674524120986462, 0.12223102897405624, 0.08496114611625671, 0.16612081229686737, 0.08873844146728516, 0.060787543654441833, 0.03336500748991966, 0.12154930084943771, 0.4316272437572479, 0.4477718770503998, 0.628478467464447]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli shekel.\n\nAnother fact: The currency used in the", "output_ids": [15907, 1340, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 14101, 33818, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 51.85838317871094, 14.148618698120117, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29874420166016, 60.48415756225586, 29.539417266845703, 129.43148803710938, 389.2738037109375, 483.4374084472656, 665.1746215820312]], "actual_norm": [[96.7635498046875, 51.8416862487793, 14.200060844421387, 137.38180541992188, 87.7329330444336, 175.22186279296875, 96.3143539428711, 60.51323318481445, 29.580154418945312, 129.47572326660156, 389.28924560546875, 483.423095703125, 665.1248779296875]], "relative_norm": [[0.0061181169003248215, 0.06623569130897522, 0.012291518040001392, 0.1213742196559906, 0.08250189572572708, 0.1623738706111908, 0.09298594295978546, 0.0608171783387661, 0.030776070430874825, 0.12026281654834747, 0.42893460392951965, 0.43469539284706116, 0.6415978670120239]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 1340, 6778, 2932, 3639, 17692, 36266, 7855, 78252, 30023], "first_top_logits": [26.625, 22.0, 21.0, 20.625, 19.625, 19.25, 19.25, 19.25, 18.875, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.4119245419278741, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/readout", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 4837.5, "target_rank": 14117.5, "source_logit": 0.66796875, "target_logit": -1.1484375, "top_ids": [8330, 11413, 45418, 38352, 15973, 7513, 4505, 3693, 3001, 7065, 7748, 28453], "top_tokens": [" Australia", " currency", ".currency", "uenta", "currency", " European", " Europe", " American", "ica", "ican", " Indian", " Currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 1787.0, "target_rank": 18971.5, "source_logit": 2.109375, "target_logit": -1.6953125, "top_ids": [11413, 15973, 96694, 5088, 3693, 53661, 45418, 28453, 26321, 42189, 3001, 34710], "top_tokens": [" currency", "currency", "onis", "urrency", " American", "cla", ".currency", " Currency", "Currency", "etz", "ica", "_currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 464.0, "target_rank": 1306.0, "source_logit": 13.875, "target_logit": 11.25, "top_ids": [11413, 35602, 90475, 8330, 7513, 22956, 45418, 32488, 3263, 34274, 85466, 10465], "top_tokens": [" currency", " currencies", "(currency", " Australia", " European", "\".\n", ".currency", " euros", "\".", " Caribbean", " Euros", ".\","]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 92.0, "target_rank": 1232.5, "source_logit": 15.0, "target_logit": 10.875, "top_ids": [11413, 7214, 35602, 3220, 47412, 65001, 37198, 13224, 21130, 17692, 7855, 16675], "top_tokens": [" currency", " foreign", " currencies", " money", " foreigners", "\"\u2014", " Commonwealth", " Britain", " parliament", " dollar", " British", " nations"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 4701.0, "target_rank": 17492.5, "source_logit": 0.53515625, "target_logit": -1.625, "top_ids": [21251, 3776, 28453, 12751, 3731, 56537, 5737, 603, 55964, 21525, 7748, 7513], "top_tokens": ["onia", " One", " Currency", " Roman", " Red", " _______,", " Group", "one", "\u2011", " Ring", " Indian", " European"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 3990.5, "target_rank": 11471.5, "source_logit": 0.2353515625, "target_logit": -1.453125, "top_ids": [7179, 11413, 479, 364, 11766, 356, 6489, 362, 2932, 4716, 18174, 3639], "top_tokens": [" International", " currency", " G", " '", " Standard", " C", " international", " A", " She", " Bar", "oret", " United"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 2245.0, "target_rank": 7774.0, "source_logit": 12.6875, "target_logit": 10.375, "top_ids": [11413, 35602, 13867, 5313, 3283, 3146, 30176, 5837, 28453, 32679, 56818, 43895], "top_tokens": [" currency", " currencies", " county", " national", " city", " country", " counties", " countries", " Currency", " citizenship", " municipality", " tariffs"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 493.0, "target_rank": 2621.0, "source_logit": 13.9375, "target_logit": 10.8125, "top_ids": [11413, 37198, 35602, 105492, 7513, 7214, 17672, 6489, 5837, 7855, 5313, 103408], "top_tokens": [" currency", " Commonwealth", " currencies", "\u4e2d\u534e\u4eba\u6c11", " European", " foreign", " euro", " international", " countries", " British", " national", "\u6b27\u5143"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3609.5, "target_rank": 8993.0, "source_logit": 0.5546875, "target_logit": -0.91796875, "top_ids": [3693, 4787, 4229, 89478, 279, 2598, 2240, 3639, 5434, 271, 15503, 6364], "top_tokens": [" American", " North", " Ge", "\u2026\n", " the", " called", " None", " United", "\u2026\n\n", "\n\n", " primarily", " English"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 579.5, "target_rank": 10122.5, "source_logit": 3.0625, "target_logit": -1.296875, "top_ids": [1852, 3693, 271, 5938, 4102, 3639, 4787, 479, 362, 7179, 547, 2274], "top_tokens": [" same", " American", "\n\n", " German", "\u00a0", " United", " North", " G", " A", " International", " U", " US"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 10807.0, "target_rank": 14889.0, "source_logit": 5.09375, "target_logit": 4.3125, "top_ids": [11413, 104142, 103408, 35602, 28453, 113883, 32488, 115160, 101237, 26321, 116866, 90475], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " currencies", " Currency", "\u672c\u56fd", " euros", "\u5176\u4ed6\u56fd\u5bb6", "\u7f8e\u5143", "Currency", "\u76f4\u8f96\u5e02", "(currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 7704.5, "target_rank": 27457.0, "source_logit": 1.4140625, "target_logit": -1.3828125, "top_ids": [104142, 101237, 106333, 103408, 116866, 110065, 101406, 112397, 35602, 106127, 113883, 109442], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", "\u5916\u6c47", "\u6b27\u5143", "\u76f4\u8f96\u5e02", "\u6c47\u7387", "\u8d27\u5e01", "\u56fd\u5bb6\u6807\u51c6", " currencies", "\u6b27\u76df", "\u672c\u56fd", "\u8d22\u653f\u90e8"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 39.0, "target_rank": 2937.0, "source_logit": 14.875, "target_logit": 3.09375, "top_ids": [11413, 11192, 28453, 104142, 17692, 80189, 35602, 103408, 32488, 85466, 19551, 42364], "top_tokens": [" currency", " dollars", " Currency", "\u4eba\u6c11\u5e01", " dollar", " Dollars", " currencies", "\u6b27\u5143", " euros", " Euros", " USD", " Dollar"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 16.0, "target_rank": 2093.5, "source_logit": 18.625, "target_logit": 3.140625, "top_ids": [11413, 104142, 17692, 15907, 28453, 11192, 35602, 19551, 36266, 3639, 80189, 42364], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollar", " Israeli", " Currency", " dollars", " currencies", " USD", " Hebrew", " United", " Dollars", " Dollar"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 74.5, "target_rank": 2721.5, "source_logit": 21.5, "target_logit": 8.4375, "top_ids": [11413, 104142, 11192, 35602, 28453, 103408, 80189, 101237, 17692, 32488, 112455, 101406], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " currencies", " Currency", "\u6b27\u5143", " Dollars", "\u7f8e\u5143", " dollar", " euros", "\u82f1\u9551", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 47.5, "target_rank": 2636.0, "source_logit": 18.625, "target_logit": 5.3125, "top_ids": [104142, 35602, 11413, 101237, 28453, 17692, 103408, 101406, 11192, 80189, 26321, 34710], "top_tokens": ["\u4eba\u6c11\u5e01", " currencies", " currency", "\u7f8e\u5143", " Currency", " dollar", "\u6b27\u5143", "\u8d27\u5e01", " dollars", " Dollars", "Currency", "_currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 12.5, "target_rank": 6198.5, "source_logit": 20.375, "target_logit": 6.28125, "top_ids": [279, 15907, 2598, 1340, 3881, 2932, 7855, 11192, 3639, 2274, 42364, 19551], "top_tokens": [" the", " Israeli", " called", " she", " known", " She", " British", " dollars", " United", " US", " Dollar", " USD"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 8.0, "target_rank": 8760.0, "source_logit": 21.875, "target_logit": 3.921875, "top_ids": [15907, 1340, 2932, 1532, 36266, 2274, 17692, 6778, 42364, 7855, 19461, 809], "top_tokens": [" Israeli", " she", " She", " New", " Hebrew", " US", " dollar", " Israel", " Dollar", " British", " Euro", " Y"]}], "source_token": 6778, "target_token": 23190, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Tel", " Aviv", " is", " the"], "seconds": 1.17373187514022}
{"run_id": "eval-24/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1764522790908813, "target_logp": -14.774893760681152, "log_odds": -13.598441481590271, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.013384844176471233, -1.1604582071304321, -0.002609302755445242]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-14.325884819030762, -0.44859686493873596, -0.00041237910045310855]}}, "score_trace": {"source": [], "target": []}, "output": " Israeli Shekel (I.S.) and the country's capital", "output_ids": [15907, 2932, 17659, 320, 40, 808, 6138, 323, 279, 3146, 594, 6722], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [], "first_top_ids": [15907, 1340, 36266, 2932, 1532, 2274, 17692, 6778, 42364, 7855], "first_top_logits": [29.0, 23.0, 22.75, 22.75, 22.75, 22.375, 22.125, 21.875, 21.375, 21.125], "source_correct": true, "target_correct": false, "seconds": 0.5171941658481956, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.1764522790908813, "target_logp": -14.774893760681152, "log_odds": -13.598441481590271, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.013384844176471233, -1.1604582071304321, -0.002609302755445242]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-14.325884819030762, -0.44859686493873596, -0.00041237910045310855]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli Shekel (I.S.) and the country's capital", "output_ids": [15907, 2932, 17659, 320, 40, 808, 6138, 323, 279, 3146, 594, 6722], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 1340, 36266, 2932, 1532, 2274, 17692, 6778, 42364, 7855], "first_top_logits": [29.0, 23.0, 22.75, 22.75, 22.75, 22.375, 22.125, 21.875, 21.375, 21.125], "source_correct": true, "target_correct": false, "seconds": 0.4372442909516394, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -32.97200393676758, "target_logp": -0.39755314588546753, "log_odds": 32.57445079088211, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-31.65458106994629, -1.3141263723373413, -0.0032974891364574432]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-0.09989345073699951, -0.2973053455352783, -0.0003543464408721775]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.88777923583984, 52.93351364135742, 12.358449935913086, 138.8209228515625, 90.29010772705078, 179.620361328125, 92.13438415527344, 60.251670837402344, 31.98891258239746, 129.26611328125, 422.021728515625, 653.7356567382812, 693.0474243164062]], "relative_norm": [[0.006125971674919128, 0.06663834303617477, 0.010692649520933628, 0.1222827211022377, 0.08501340448856354, 0.16614240407943726, 0.08873146027326584, 0.06073872745037079, 0.03335065767168999, 0.12071733176708221, 0.46646761894226074, 0.5921235680580139, 0.6689441204071045]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.88777923583984, 52.93351364135742, 12.358449935913086, 138.8209228515625, 90.29010772705078, 179.620361328125, 92.13438415527344, 60.251670837402344, 31.98891258239746, 129.26611328125, 422.021728515625, 653.7356567382812, 693.0474243164062]], "relative_norm": [[0.006125971674919128, 0.06663834303617477, 0.010692649520933628, 0.1222827211022377, 0.08501340448856354, 0.16614240407943726, 0.08873146027326584, 0.06073872745037079, 0.03335065767168999, 0.12071733176708221, 0.46646761894226074, 0.5921235680580139, 0.6689441204071045]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the name of the country", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 829, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.148619651794434, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539419174194336, 130.31475830078125, 417.6007995605469, 657.7894897460938, 695.8855590820312]], "actual_norm": [[96.88777923583984, 51.86480712890625, 14.185067176818848, 137.3540496826172, 87.75621032714844, 175.22848510742188, 96.29241943359375, 60.45827102661133, 29.59929084777832, 130.32803344726562, 417.5784606933594, 657.8222045898438, 695.8521728515625]], "relative_norm": [[0.006125971674919128, 0.06626523286104202, 0.012278539128601551, 0.12134969234466553, 0.08252378553152084, 0.16238000988960266, 0.09296476095914841, 0.060761939734220505, 0.030795980244874954, 0.12112686038017273, 0.4616764783859253, 0.5949370265007019, 0.6735857725143433]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 23190, 5052, 84574, 110055, 4492, 19461, 37088, 220, 17672], "first_top_logits": [29.75, 27.375, 25.25, 23.375, 22.75, 21.625, 21.625, 21.0, 20.75, 20.5], "source_correct": false, "target_correct": true, "seconds": 0.40942679112777114, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1062567234039307, "target_logp": -15.641779899597168, "log_odds": -13.535523176193237, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.609353244304657, -1.4939618110656738, -0.0029416880570352077]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-15.234353065490723, -0.4071561098098755, -0.00027104519540444016]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.29171371459961, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.1296615600586, 60.29090881347656, 31.99578094482422, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.88272094726562, 53.01782989501953, 12.327152252197266, 138.79266357421875, 90.29666137695312, 179.614501953125, 92.15062713623047, 60.316429138183594, 32.04558563232422, 129.32945251464844, 422.0733947753906, 653.7939453125, 693.07275390625]], "relative_norm": [[0.006125651765614748, 0.06674448400735855, 0.010665570385754108, 0.12225782871246338, 0.08501957356929779, 0.1661369800567627, 0.08874709904193878, 0.060804009437561035, 0.03340974450111389, 0.12077648192644119, 0.46652474999427795, 0.5921763777732849, 0.6689685583114624]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.29171371459961, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.1296615600586, 60.29090881347656, 31.99578094482422, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.88272094726562, 53.01782989501953, 12.327152252197266, 138.79266357421875, 90.29666137695312, 179.614501953125, 92.15062713623047, 60.316429138183594, 32.04558563232422, 129.32945251464844, 422.0733947753906, 653.7939453125, 693.07275390625]], "relative_norm": [[0.006125651765614748, 0.06674448400735855, 0.010665570385754108, 0.12225782871246338, 0.08501957356929779, 0.1661369800567627, 0.08874709904193878, 0.060804009437561035, 0.03340974450111389, 0.12077648192644119, 0.46652474999427795, 0.5921763777732849, 0.6689685583114624]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli Shekel (ILS).\n\nFact: The currency used", "output_ids": [15907, 2932, 17659, 320, 1715, 50, 3593, 17417, 25, 576, 11413, 1483], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.14862060546875, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539417266845703, 130.31475830078125, 417.6007995605469, 657.7894897460938, 695.8855590820312]], "actual_norm": [[96.88272094726562, 51.881771087646484, 14.216714859008789, 137.41973876953125, 87.73492431640625, 175.16087341308594, 96.29576110839844, 60.52335739135742, 29.563894271850586, 130.313232421875, 417.62841796875, 657.8006591796875, 695.78662109375]], "relative_norm": [[0.006125651765614748, 0.06628690659999847, 0.01230593305081129, 0.12140773236751556, 0.0825037732720375, 0.16231735050678253, 0.09296798706054688, 0.0608273521065712, 0.03075915388762951, 0.12111309915781021, 0.4617317020893097, 0.5949175357818604, 0.6735222935676575]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 1965, 17692, 6778, 36266, 2274, 1140, 11413, 6633, 19551], "first_top_logits": [26.0, 25.625, 21.875, 21.5, 21.125, 20.875, 20.125, 19.5, 19.5, 19.375], "source_correct": true, "target_correct": false, "seconds": 0.46818108297884464, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.367382764816284, "target_logp": -39.57859802246094, "log_odds": -37.21121525764465, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.9120604991912842, -1.4530166387557983, -0.00230582756921649]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-38.97455978393555, -0.6036826372146606, -0.00035589560866355896]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.90357208251953, 52.93779373168945, 12.352404594421387, 138.827392578125, 90.2739486694336, 179.63320922851562, 92.09620666503906, 60.278045654296875, 31.991329193115234, 129.24583435058594, 422.01995849609375, 653.8522338867188, 693.0940551757812]], "relative_norm": [[0.006126970052719116, 0.06664372980594635, 0.010687419213354588, 0.12228842079639435, 0.08499819040298462, 0.16615428030490875, 0.08869469165802002, 0.06076531484723091, 0.033353179693222046, 0.12069839239120483, 0.46646568179130554, 0.5922291874885559, 0.6689891219139099]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63893127441406, 92.12966918945312, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.90357208251953, 52.93779373168945, 12.352404594421387, 138.827392578125, 90.2739486694336, 179.63320922851562, 92.09620666503906, 60.278045654296875, 31.991329193115234, 129.24583435058594, 422.01995849609375, 653.8522338867188, 693.0940551757812]], "relative_norm": [[0.006126970052719116, 0.06664372980594635, 0.010687419213354588, 0.12228842079639435, 0.08499819040298462, 0.16615428030490875, 0.08869469165802002, 0.06076531484723091, 0.033353179693222046, 0.12069839239120483, 0.46646568179130554, 0.5922291874885559, 0.6689891219139099]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israel New Shekel.\n\nAnother fact: The currency used in", "output_ids": [6778, 1532, 2932, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862060546875, 51.85838317871094, 14.148619651794434, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29875183105469, 60.484161376953125, 29.539419174194336, 130.31475830078125, 417.6007995605469, 657.7894897460938, 695.8855590820312]], "actual_norm": [[96.90357208251953, 51.88034439086914, 14.18665599822998, 137.36122131347656, 87.76414489746094, 175.18882751464844, 96.31395721435547, 60.506248474121094, 29.63409423828125, 130.32772827148438, 417.5804138183594, 657.7998046875, 695.8984375]], "relative_norm": [[0.006126970052719116, 0.06628508120775223, 0.012279914692044258, 0.12135603278875351, 0.08253125101327896, 0.16234326362609863, 0.09298555552959442, 0.060810159891843796, 0.030832191929221153, 0.12112657725811005, 0.4616786539554596, 0.5949167609214783, 0.6736305356025696]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6778, 15907, 2932, 1340, 36266, 17692, 501, 13350, 42364, 98124], "first_top_logits": [27.25, 26.75, 23.0, 21.75, 21.25, 19.0, 18.5, 18.375, 17.75, 17.625], "source_correct": true, "target_correct": false, "seconds": 0.44900841685011983, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2990814447402954, "target_logp": -12.378274917602539, "log_odds": -11.079193472862244, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.10399319231510162, -1.1908525228500366, -0.004235701635479927]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-11.66649341583252, -0.7114489078521729, -0.0003326578007545322]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.63893127441406, 92.1296615600586, 60.2909049987793, 31.995777130126953, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.115966796875]], "actual_norm": [[96.83702087402344, 52.98702621459961, 12.302899360656738, 138.7931365966797, 90.36840057373047, 179.61146545410156, 92.21683502197266, 60.272708892822266, 31.98310089111328, 129.30484008789062, 421.9409484863281, 653.6932373046875, 693.1948852539062]], "relative_norm": [[0.0061227623373270035, 0.06670571118593216, 0.010644586756825447, 0.1222582459449768, 0.0850871205329895, 0.16613417863845825, 0.08881086111068726, 0.06075993552803993, 0.03334460034966469, 0.12075349688529968, 0.4663783311843872, 0.5920851826667786, 0.6690864562988281]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.63893127441406, 92.1296615600586, 60.2909049987793, 31.995777130126953, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.115966796875]], "actual_norm": [[96.83702087402344, 52.98702621459961, 12.302899360656738, 138.7931365966797, 90.36840057373047, 179.61146545410156, 92.21683502197266, 60.272708892822266, 31.98310089111328, 129.30484008789062, 421.9409484863281, 653.6932373046875, 693.1948852539062]], "relative_norm": [[0.0061227623373270035, 0.06670571118593216, 0.010644586756825447, 0.1222582459449768, 0.0850871205329895, 0.16613417863845825, 0.08881086111068726, 0.06075993552803993, 0.03334460034966469, 0.12075349688529968, 0.4663783311843872, 0.5920851826667786, 0.6690864562988281]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli Shekel (ILS).\n\nFact: The currency used", "output_ids": [15907, 2932, 17659, 320, 1715, 50, 3593, 17417, 25, 576, 11413, 1483], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 51.85837936401367, 14.148618698120117, 137.3745880126953, 87.7684555053711, 175.21165466308594, 96.29875183105469, 60.48415756225586, 29.539417266845703, 130.3147430419922, 417.60076904296875, 657.7894897460938, 695.885498046875]], "actual_norm": [[96.83702087402344, 51.840091705322266, 14.142302513122559, 137.46112060546875, 87.87090301513672, 175.23001098632812, 96.34798431396484, 60.480533599853516, 29.524280548095703, 130.42306518554688, 417.5762023925781, 657.7503662109375, 695.856201171875]], "relative_norm": [[0.0061227623373270035, 0.06623365730047226, 0.01224152185022831, 0.12144429236650467, 0.08263164013624191, 0.16238142549991608, 0.09301841259002686, 0.06078431382775307, 0.030717937275767326, 0.12121517956256866, 0.4616739749908447, 0.5948720574378967, 0.6735896468162537]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 7855, 19461, 2932, 3639, 17692, 23734, 17672, 1340, 422], "first_top_logits": [25.125, 21.875, 21.625, 21.0, 19.75, 19.5, 19.375, 19.375, 19.25, 19.0], "source_correct": true, "target_correct": false, "seconds": 0.46669100038707256, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.4647021293640137, "target_logp": -12.346878051757812, "log_odds": -9.882175922393799, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-1.0461572408676147, -1.415945053100586, -0.0025997909251600504]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-12.046156883239746, -0.3004874885082245, -0.0002335037279408425]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63894653320312, 92.12967681884766, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160888671875]], "actual_norm": [[96.91927337646484, 52.99755096435547, 12.352665901184082, 138.75584411621094, 90.33192443847656, 179.64727783203125, 92.12714385986328, 60.311954498291016, 31.992265701293945, 129.32496643066406, 422.031494140625, 653.7476806640625, 692.98486328125]], "relative_norm": [[0.006127962842583656, 0.06671895831823349, 0.010687645524740219, 0.12222539633512497, 0.08505277335643768, 0.16616730391979218, 0.08872448652982712, 0.0607994981110096, 0.03335415571928024, 0.12077229470014572, 0.4664784371852875, 0.5921344757080078, 0.668883740901947]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 52.96437454223633, 12.291714668273926, 138.7812042236328, 90.27318572998047, 179.63894653320312, 92.12967681884766, 60.29090881347656, 31.995779037475586, 129.256591796875, 422.013427734375, 653.7408447265625, 693.1160888671875]], "actual_norm": [[96.91927337646484, 52.99755096435547, 12.352665901184082, 138.75584411621094, 90.33192443847656, 179.64727783203125, 92.12714385986328, 60.311954498291016, 31.992265701293945, 129.32496643066406, 422.031494140625, 653.7476806640625, 692.98486328125]], "relative_norm": [[0.006127962842583656, 0.06671895831823349, 0.010687645524740219, 0.12222539633512497, 0.08505277335643768, 0.16616730391979218, 0.08872448652982712, 0.0607994981110096, 0.03335415571928024, 0.12077229470014572, 0.4664784371852875, 0.5921344757080078, 0.668883740901947]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli New Shekel.\n\nAnother fact: The currency used in", "output_ids": [15907, 1532, 2932, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89862823486328, 51.85838317871094, 14.14862060546875, 137.3745880126953, 87.7684555053711, 175.21168518066406, 96.29875183105469, 60.484161376953125, 29.539419174194336, 130.31475830078125, 417.6007995605469, 657.78955078125, 695.8855590820312]], "actual_norm": [[96.91927337646484, 51.87762451171875, 14.202552795410156, 137.39434814453125, 87.7420425415039, 175.18060302734375, 96.27130126953125, 60.55527877807617, 29.561262130737305, 130.3546905517578, 417.5984802246094, 657.8975219726562, 695.8118286132812]], "relative_norm": [[0.006127962842583656, 0.06628160923719406, 0.012293674983084202, 0.12138529866933823, 0.08251046389341354, 0.16233563423156738, 0.09294437617063522, 0.06085943430662155, 0.03075641393661499, 0.1211516335606575, 0.46169862151145935, 0.5950051546096802, 0.6735467314720154]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [19461, 15907, 1532, 17672, 17692, 36266, 42364, 13, 3639, 6778], "first_top_logits": [23.875, 23.875, 22.75, 22.625, 21.75, 19.75, 19.5, 19.25, 18.875, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.450917084235698, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-24/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-24", "split": "eval", "country": "Israel", "currency": "Israeli shekel", "aliases": ["shekel", "ILS"], "city": "Tel Aviv", "target_country": "Sweden", "target_currency": "Swedish krona", "target_aliases": ["krona", "SEK"], "prompt": "Fact: The currency used in the country containing Tel Aviv is the", "source_continuation": " Israeli shekel", "target_continuation": " Swedish krona"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0954227447509766, "target_logp": -14.36636734008789, "log_odds": -13.270944595336914, "token_scores": {"source": {"ids": [15907, 1340, 17659], "log_probs": [-0.02499411441385746, -1.066681146621704, -0.0037474660202860832]}, "target": {"ids": [30109, 87932, 64], "log_probs": [-13.899993896484375, -0.4661174714565277, -0.00025567124248482287]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.638916015625, 92.1296615600586, 60.2909049987793, 31.995777130126953, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.7635498046875, 52.94948196411133, 12.33750057220459, 138.76223754882812, 90.23460388183594, 179.59701538085938, 92.14163208007812, 60.30009841918945, 32.0026741027832, 129.22886657714844, 422.0201721191406, 653.6632690429688, 693.1069946289062]], "relative_norm": [[0.0061181169003248215, 0.06665844470262527, 0.010674524120986462, 0.12223102897405624, 0.08496114611625671, 0.16612081229686737, 0.08873844146728516, 0.060787543654441833, 0.03336500748991966, 0.12068254500627518, 0.46646592020988464, 0.592058002948761, 0.6690016388893127]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 52.96437072753906, 12.291714668273926, 138.7812042236328, 90.27317810058594, 179.638916015625, 92.1296615600586, 60.2909049987793, 31.995777130126953, 129.256591796875, 422.0133972167969, 653.7408447265625, 693.1160278320312]], "actual_norm": [[96.7635498046875, 52.94948196411133, 12.33750057220459, 138.76223754882812, 90.23460388183594, 179.59701538085938, 92.14163208007812, 60.30009841918945, 32.0026741027832, 129.22886657714844, 422.0201721191406, 653.6632690429688, 693.1069946289062]], "relative_norm": [[0.0061181169003248215, 0.06665844470262527, 0.010674524120986462, 0.12223102897405624, 0.08496114611625671, 0.16612081229686737, 0.08873844146728516, 0.060787543654441833, 0.03336500748991966, 0.12068254500627518, 0.46646592020988464, 0.592058002948761, 0.6690016388893127]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Israeli Shekel.\n\nAnother fact: The currency used in the", "output_ids": [15907, 2932, 17659, 382, 14037, 2097, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 22836, 65847, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[96.89861297607422, 51.85838317871094, 14.148618698120117, 137.3745880126953, 87.7684555053711, 175.211669921875, 96.29874420166016, 60.48415756225586, 29.539417266845703, 130.3147430419922, 417.6007995605469, 657.7894287109375, 695.8855590820312]], "actual_norm": [[96.7635498046875, 51.8416862487793, 14.200060844421387, 137.38180541992188, 87.7329330444336, 175.22186279296875, 96.3143539428711, 60.51323318481445, 29.580154418945312, 130.38816833496094, 417.613525390625, 657.7998657226562, 695.8441162109375]], "relative_norm": [[0.0061181169003248215, 0.06623569130897522, 0.012291518040001392, 0.1213742196559906, 0.08250189572572708, 0.1623738706111908, 0.09298594295978546, 0.0608171783387661, 0.030776070430874825, 0.12118274718523026, 0.4617152512073517, 0.594916820526123, 0.6735779643058777]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [15907, 17692, 1340, 42364, 809, 6778, 2932, 3639, 36266, 2274], "first_top_logits": [26.625, 21.375, 21.375, 21.0, 20.625, 20.5, 20.375, 19.625, 18.75, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.4313291250728071, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/readout", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 4460.0, "target_rank": 1288.5, "source_logit": 0.62109375, "target_logit": 2.34375, "top_ids": [8330, 4505, 7513, 38352, 11413, 3001, 19312, 45418, 6364, 4787, 49029, 15973], "top_tokens": [" Australia", " Europe", " European", "uenta", " currency", "ica", "urname", ".currency", " English", " North", "ctica", "currency"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 9240.5, "target_rank": 467.5, "source_logit": -0.54296875, "target_logit": 3.578125, "top_ids": [11413, 15973, 96694, 3001, 53661, 28453, 6364, 4787, 3693, 18174, 28130, 42189], "top_tokens": [" currency", "currency", "onis", "ica", "cla", " Currency", " English", " North", " American", "oret", "ija", "etz"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 162.0, "target_rank": 232.5, "source_logit": 14.625, "target_logit": 13.875, "top_ids": [11413, 8330, 35602, 91984, 12463, 38664, 26138, 85466, 32488, 13369, 29973, 7513], "top_tokens": [" currency", " Australia", " currencies", " licences", " colour", " recognised", " colours", " Euros", " euros", " Australian", " licence", " European"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 118.5, "target_rank": 163.5, "source_logit": 15.1875, "target_logit": 14.6875, "top_ids": [11413, 13224, 7855, 7214, 21130, 87598, 35602, 17672, 37198, 8330, 74508, 47412], "top_tokens": [" currency", " Britain", " British", " foreign", " parliament", " canadian", " currencies", " euro", " Commonwealth", " Australia", "\u00bb.", " foreigners"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 8715.0, "target_rank": 3855.5, "source_logit": -0.609375, "target_logit": 0.6484375, "top_ids": [21251, 3776, 12751, 5737, 3731, 4229, 28453, 21525, 19833, 55964, 9127, 11426], "top_tokens": ["onia", " One", " Roman", " Group", " Red", " Ge", " Currency", " Ring", " Amb", "\u2011", " Rock", " Land"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 4809.5, "target_rank": 2918.5, "source_logit": -0.0673828125, "target_logit": 0.68359375, "top_ids": [364, 7179, 11766, 479, 2932, 11413, 6489, 4716, 467, 356, 21251, 18174], "top_tokens": [" '", " International", " Standard", " G", " She", " currency", " international", " Bar", " W", " C", "onia", "oret"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 2405.5, "target_rank": 1249.0, "source_logit": 12.4375, "target_logit": 13.5625, "top_ids": [11413, 35602, 13867, 30176, 3146, 29973, 91984, 5837, 3283, 5313, 32488, 72836], "top_tokens": [" currency", " currencies", " county", " counties", " country", " licence", " licences", " countries", " city", " national", " euros", " postcode"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 486.0, "target_rank": 104.5, "source_logit": 14.1875, "target_logit": 16.875, "top_ids": [37198, 7513, 17672, 11413, 5837, 7855, 35602, 6489, 103408, 6424, 4505, 32488], "top_tokens": [" Commonwealth", " European", " euro", " currency", " countries", " British", " currencies", " international", "\u6b27\u5143", " UK", " Europe", " euros"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 9876.0, "target_rank": 3992.5, "source_logit": -0.96875, "target_logit": 0.54296875, "top_ids": [4787, 3693, 4229, 279, 271, 89478, 6364, 5434, 3639, 220, 46427, 1083], "top_tokens": [" North", " American", " Ge", " the", "\n\n", "\u2026\n", " English", "\u2026\n\n", " United", " ", "/is", " also"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 5136.0, "target_rank": 9474.5, "source_logit": -0.2119140625, "target_logit": -1.3046875, "top_ids": [1852, 4787, 271, 5938, 4102, 4882, 3639, 479, 3693, 362, 220, 5055], "top_tokens": [" same", " North", "\n\n", " German", "\u00a0", " South", " United", " G", " American", " A", " ", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 25135.5, "target_rank": 6561.0, "source_logit": -0.71875, "target_logit": 2.0, "top_ids": [104142, 103408, 113883, 35602, 11413, 112455, 110546, 115160, 32488, 28453, 106127, 106333], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u672c\u56fd", " currencies", " currency", "\u82f1\u9551", "\u82f1\u683c\u5170", "\u5176\u4ed6\u56fd\u5bb6", " euros", " Currency", "\u6b27\u76df", "\u5916\u6c47"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 9573.0, "target_rank": 1418.5, "source_logit": -0.609375, "target_logit": 2.515625, "top_ids": [104142, 103408, 112397, 116866, 106127, 113883, 106333, 116588, 99599, 109442, 101237, 110065], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u6b27\u76df", "\u672c\u56fd", "\u5916\u6c47", "\u6210\u5458\u56fd", "\u56fd\u5bb6", "\u8d22\u653f\u90e8", "\u7f8e\u5143", "\u6c47\u7387"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 5.5, "target_rank": 453.0, "source_logit": 22.0, "target_logit": 7.375, "top_ids": [30109, 11413, 110055, 32488, 103408, 23190, 85466, 11192, 112455, 104142, 44621, 43680], "top_tokens": [" Swedish", " currency", "\u745e\u5178", " euros", "\u6b27\u5143", " Sweden", " Euros", " dollars", "\u82f1\u9551", "\u4eba\u6c11\u5e01", " Norwegian", " Danish"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 686.0, "source_logit": 27.75, "target_logit": 5.625, "top_ids": [30109, 110055, 23190, 84574, 44621, 43680, 57853, 52082, 96871, 63366, 118846, 86953], "top_tokens": [" Swedish", "\u745e\u5178", " Sweden", " Scandinavian", " Norwegian", " Danish", " Finnish", " Stockholm", "Sweden", " Nordic", "\u82ac\u5170", " svenska"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 21.5, "target_rank": 271.5, "source_logit": 27.375, "target_logit": 13.6875, "top_ids": [11413, 103408, 104142, 32488, 112455, 110055, 11192, 35602, 28453, 30109, 101406, 85466], "top_tokens": [" currency", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " euros", "\u82f1\u9551", "\u745e\u5178", " dollars", " currencies", " Currency", " Swedish", "\u8d27\u5e01", " Euros"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 3.0, "target_rank": 306.5, "source_logit": 28.0, "target_logit": 9.25, "top_ids": [110055, 30109, 23190, 104142, 96871, 84574, 57853, 63366, 44621, 103408, 52082, 118846], "top_tokens": ["\u745e\u5178", " Swedish", " Sweden", "\u4eba\u6c11\u5e01", "Sweden", " Scandinavian", " Finnish", " Nordic", " Norwegian", "\u6b27\u5143", " Stockholm", "\u82ac\u5170"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 33.5, "target_rank": 6814.5, "source_logit": 17.5, "target_logit": 4.8125, "top_ids": [279, 87932, 30109, 96560, 5052, 17672, 3881, 19461, 32488, 37088, 220, 1083], "top_tokens": [" the", " kron", " Swedish", " Kron", " SE", " euro", " known", " Euro", " euros", " EUR", " ", " also"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 18.5, "target_rank": 8679.5, "source_logit": 17.75, "target_logit": 2.703125, "top_ids": [30109, 87932, 96560, 5052, 17672, 19461, 110055, 44621, 84574, 16926, 63366, 4492], "top_tokens": [" Swedish", " kron", " Kron", " SE", " euro", " Euro", "\u745e\u5178", " Norwegian", " Scandinavian", " Northern", " Nordic", " Sw"]}], "source_token": 23190, "target_token": 29121, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Go", "then", "burg", " is", " the"], "seconds": 1.1486708750016987}
{"run_id": "eval-25/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.33078283071517944, "target_logp": -14.330254554748535, "log_odds": -13.999471724033356, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.06685933470726013, -0.2622985541820526, -0.0016249322798103094]}, "target": {"ids": [29691, 43479], "log_probs": [-13.566859245300293, -0.7633952498435974]}}, "score_trace": {"source": [], "target": []}, "output": " Swedish krona.\n\nFact: The currency used in the country", "output_ids": [30109, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [], "first_top_ids": [30109, 87932, 96560, 5052, 17672, 19461, 110055, 44621, 84574, 16926], "first_top_logits": [28.25, 25.5, 23.625, 22.75, 22.25, 21.625, 21.5, 20.75, 20.0, 19.75], "source_correct": true, "target_correct": false, "seconds": 0.5166162080131471, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.33078283071517944, "target_logp": -14.330254554748535, "log_odds": -13.999471724033356, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.06685933470726013, -0.2622985541820526, -0.0016249322798103094]}, "target": {"ids": [29691, 43479], "log_probs": [-13.566859245300293, -0.7633952498435974]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.\n\nFact: The currency used in the country", "output_ids": [30109, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 87932, 96560, 5052, 17672, 19461, 110055, 44621, 84574, 16926], "first_top_logits": [28.25, 25.5, 23.625, 22.75, 22.25, 21.625, 21.5, 20.75, 20.0, 19.75], "source_correct": true, "target_correct": false, "seconds": 0.439046666957438, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -32.6331901550293, "target_logp": -3.2268660068511963, "log_odds": 29.4063241481781, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-32.22980880737305, -0.40152373909950256, -0.0018585327779874206]}, "target": {"ids": [29691, 43479], "log_probs": [-2.464184284210205, -0.7626817226409912]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.685401916503906, 55.04582977294922, 8.306366920471191, 374.4843444824219, 190.5871124267578, 195.06292724609375, 246.84976196289062, 288.7644348144531, 348.5726013183594, 320.1703796386719, 374.10198974609375, 335.2754211425781, 740.9929809570312, 1120.080078125]], "relative_norm": [[0.0029518010560423136, 0.06929755210876465, 0.007186748553067446, 0.3298707604408264, 0.17944888770580292, 0.180426225066185, 0.23773252964019775, 0.2910987138748169, 0.3634110987186432, 0.29518842697143555, 0.32750892639160156, 0.37151196599006653, 0.6708166599273682, 1.095166563987732]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.685401916503906, 55.04582977294922, 8.306366920471191, 374.4843444824219, 190.5871124267578, 195.06292724609375, 246.84976196289062, 288.7644348144531, 348.5726013183594, 320.1703796386719, 374.10198974609375, 335.2754211425781, 740.9929809570312, 1120.080078125]], "relative_norm": [[0.0029518010560423136, 0.06929755210876465, 0.007186748553067446, 0.3298707604408264, 0.17944888770580292, 0.180426225066185, 0.23773252964019775, 0.2910987138748169, 0.3634110987186432, 0.29518842697143555, 0.32750892639160156, 0.37151196599006653, 0.6708166599273682, 1.095166563987732]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Switzerland Franc.  What is the currency used in the country", "output_ids": [29121, 9694, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 320.6308288574219, 364.7038269042969, 328.0982666015625, 748.3380126953125, 1102.7435302734375]], "actual_norm": [[46.685401916503906, 54.61267852783203, 9.771011352539062, 371.4886779785156, 193.52008056640625, 196.80213928222656, 247.17041015625, 292.3385925292969, 351.5611267089844, 320.69146728515625, 364.6860046386719, 328.0551452636719, 748.310791015625, 1102.6109619140625]], "relative_norm": [[0.0029518010560423136, 0.06977605819702148, 0.008457749150693417, 0.3282032012939453, 0.18198153376579285, 0.18237179517745972, 0.23862874507904053, 0.29380694031715393, 0.36577463150024414, 0.29580825567245483, 0.3198395073413849, 0.3625207841396332, 0.6749641299247742, 1.0790629386901855]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29121, 29691, 108205, 6826, 71726, 13218, 16464, 44245, 220, 59546], "first_top_logits": [28.75, 26.5, 20.625, 16.25, 14.3125, 14.1875, 13.75, 13.625, 13.375, 13.0], "source_correct": false, "target_correct": true, "seconds": 0.42409416707232594, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.952960968017578, "target_logp": -21.772705078125, "log_odds": -15.819744110107422, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-5.63123893737793, -0.3210020661354065, -0.0007198841194622219]}, "target": {"ids": [29691, 43479], "log_probs": [-20.94373893737793, -0.8289653062820435]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.1237487792969, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.64292526245117, 55.00143051147461, 8.3364896774292, 374.4998779296875, 190.60134887695312, 195.0970458984375, 246.79107666015625, 288.7442932128906, 348.5768127441406, 320.27716064453125, 374.1273193359375, 335.28680419921875, 741.0831298828125, 1120.1180419921875]], "relative_norm": [[0.002949115354567766, 0.06924165785312653, 0.007212810683995485, 0.32988446950912476, 0.1794622838497162, 0.18045778572559357, 0.2376760095357895, 0.2910784184932709, 0.36341550946235657, 0.2952868938446045, 0.327531099319458, 0.3715245723724365, 0.6708982586860657, 1.0952036380767822]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.1237487792969, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.64292526245117, 55.00143051147461, 8.3364896774292, 374.4998779296875, 190.60134887695312, 195.0970458984375, 246.79107666015625, 288.7442932128906, 348.5768127441406, 320.27716064453125, 374.1273193359375, 335.28680419921875, 741.0831298828125, 1120.1180419921875]], "relative_norm": [[0.002949115354567766, 0.06924165785312653, 0.007212810683995485, 0.32988446950912476, 0.1794622838497162, 0.18045778572559357, 0.2376760095357895, 0.2910784184932709, 0.36341550946235657, 0.2952868938446045, 0.327531099319458, 0.3715245723724365, 0.6708982586860657, 1.0952036380767822]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Go", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 5994], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 54.57722854614258, 9.721745491027832, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 320.6308288574219, 364.7038269042969, 328.0982666015625, 748.3380126953125, 1102.7435302734375]], "actual_norm": [[46.64292526245117, 54.57379913330078, 9.746484756469727, 371.4788513183594, 193.49276733398438, 196.79794311523438, 247.14505004882812, 292.3660583496094, 351.5832824707031, 320.6741027832031, 364.7177734375, 328.08892822265625, 748.309326171875, 1102.719482421875]], "relative_norm": [[0.002949115354567766, 0.0697263851761818, 0.008436519652605057, 0.3281945288181305, 0.18195584416389465, 0.18236790597438812, 0.23860426247119904, 0.29383453726768494, 0.3657976984977722, 0.29579225182533264, 0.3198673725128174, 0.3625580966472626, 0.6749628186225891, 1.0791691541671753]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 30109, 220, 6633, 85466, 17672, 37088, 19461, 7513, 198], "first_top_logits": [28.875, 23.625, 21.75, 21.375, 20.5, 19.875, 19.75, 19.75, 19.75, 19.625], "source_correct": false, "target_correct": false, "seconds": 0.42037704214453697, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.895010948181152, "target_logp": -32.3895263671875, "log_odds": -27.494515419006348, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-4.660764217376709, -0.23262479901313782, -0.0016215997748076916]}, "target": {"ids": [29691, 43479], "log_probs": [-31.410764694213867, -0.9787625670433044]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.659061431884766, 55.05573654174805, 8.30714225769043, 374.5028076171875, 190.60870361328125, 195.0790557861328, 246.83595275878906, 288.7991638183594, 348.57391357421875, 320.21453857421875, 374.1156921386719, 335.2397155761719, 740.849365234375, 1120.1236572265625]], "relative_norm": [[0.0029501356184482574, 0.0693100243806839, 0.007187419105321169, 0.3298870325088501, 0.17946921288967133, 0.18044114112854004, 0.23771922290325165, 0.29113373160362244, 0.363412469625473, 0.29522913694381714, 0.3275209367275238, 0.37147238850593567, 0.670686662197113, 1.0952092409133911]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 320.2182312011719, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.659061431884766, 55.05573654174805, 8.30714225769043, 374.5028076171875, 190.60870361328125, 195.0790557861328, 246.83595275878906, 288.7991638183594, 348.57391357421875, 320.21453857421875, 374.1156921386719, 335.2397155761719, 740.849365234375, 1120.1236572265625]], "relative_norm": [[0.0029501356184482574, 0.0693100243806839, 0.007187419105321169, 0.3298870325088501, 0.17946921288967133, 0.18044114112854004, 0.23771922290325165, 0.29113373160362244, 0.363412469625473, 0.29522913694381714, 0.3275209367275238, 0.37147238850593567, 0.670686662197113, 1.0952092409133911]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Sweden krona.\n\nFact: The currency used in the country", "output_ids": [23190, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662845611572266, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 320.6308288574219, 364.7038269042969, 328.0982666015625, 748.3380126953125, 1102.7435302734375]], "actual_norm": [[46.659061431884766, 54.617027282714844, 9.773731231689453, 371.537109375, 193.53634643554688, 196.8165740966797, 247.10540771484375, 292.3741455078125, 351.58990478515625, 320.57757568359375, 364.7384338378906, 328.1050109863281, 748.401611328125, 1102.70654296875]], "relative_norm": [[0.0029501356184482574, 0.06978161633014679, 0.008460103534162045, 0.32824599742889404, 0.18199683725833893, 0.1823851764202118, 0.23856598138809204, 0.2938426733016968, 0.3658045828342438, 0.2957032024860382, 0.3198854923248291, 0.362575888633728, 0.6750460863113403, 1.079156517982483]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23190, 468, 730, 595, 1852, 30109, 86953, 220, 817, 773], "first_top_logits": [23.25, 20.75, 20.375, 20.125, 19.0, 19.0, 18.5, 18.375, 18.375, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.46768362494185567, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.44980090856552124, "target_logp": -15.468801498413086, "log_odds": -15.019000589847565, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.01341942511498928, -0.43509045243263245, -0.001291037304326892]}, "target": {"ids": [29691, 43479], "log_probs": [-14.638419151306152, -0.8303821086883545]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277507781982422, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.8117218017578, 288.7742004394531, 348.59930419921875, 320.21820068359375, 374.123779296875, 335.2681884765625, 741.0165405273438, 1120.0623779296875]], "actual_norm": [[46.60539627075195, 55.07643127441406, 8.313722610473633, 374.48187255859375, 190.56602478027344, 194.99717712402344, 246.8210906982422, 288.8201904296875, 348.5928039550781, 320.1693420410156, 374.0989990234375, 335.1949157714844, 740.9974975585938, 1120.085693359375]], "relative_norm": [[0.002946742344647646, 0.06933607161045074, 0.007193112745881081, 0.3298685848712921, 0.17942902445793152, 0.1803654134273529, 0.23770490288734436, 0.2911549210548401, 0.3634321689605713, 0.29518747329711914, 0.32750630378723145, 0.37142276763916016, 0.6708207130432129, 1.0951720476150513]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277507781982422, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.8117218017578, 288.7742004394531, 348.59930419921875, 320.21820068359375, 374.123779296875, 335.2681884765625, 741.0165405273438, 1120.0623779296875]], "actual_norm": [[46.60539627075195, 55.07643127441406, 8.313722610473633, 374.48187255859375, 190.56602478027344, 194.99717712402344, 246.8210906982422, 288.8201904296875, 348.5928039550781, 320.1693420410156, 374.0989990234375, 335.1949157714844, 740.9974975585938, 1120.085693359375]], "relative_norm": [[0.002946742344647646, 0.06933607161045074, 0.007193112745881081, 0.3298685848712921, 0.17942902445793152, 0.1803654134273529, 0.23770490288734436, 0.2911549210548401, 0.3634321689605713, 0.29518747329711914, 0.32750630378723145, 0.37142276763916016, 0.6708207130432129, 1.0951720476150513]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.50848388671875, 196.81228637695312, 247.119873046875, 292.3907775878906, 351.5521240234375, 320.63079833984375, 364.70379638671875, 328.0982360839844, 748.3380126953125, 1102.743408203125]], "actual_norm": [[46.60539627075195, 54.58580780029297, 9.756089210510254, 371.44281005859375, 193.45623779296875, 196.7270050048828, 247.15594482421875, 292.424560546875, 351.525146484375, 320.68511962890625, 364.7306823730469, 328.0785827636719, 748.2928466796875, 1102.7137451171875]], "relative_norm": [[0.002946742344647646, 0.06974172592163086, 0.008444832637906075, 0.32816267013549805, 0.18192149698734283, 0.18230216205120087, 0.2386147826910019, 0.29389333724975586, 0.3657371997833252, 0.29580241441726685, 0.3198786973953247, 0.3625466823577881, 0.67494797706604, 1.0791635513305664]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 17672, 19461, 110055, 4492, 32488, 5052, 220, 30, 84574], "first_top_logits": [26.625, 21.875, 20.75, 20.375, 19.5, 18.75, 18.625, 18.25, 18.0, 17.375], "source_correct": true, "target_correct": false, "seconds": 0.45220812503248453, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5173379182815552, "target_logp": -10.484041213989258, "log_odds": -9.966703295707703, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.409688264131546, -0.10617229342460632, -0.0014773417497053742]}, "target": {"ids": [29691, 43479], "log_probs": [-9.722187995910645, -0.7618528604507446]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81175231933594, 288.77423095703125, 348.5993347167969, 320.21826171875, 374.1238098144531, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.65830612182617, 55.04728698730469, 8.402240753173828, 374.5434265136719, 190.55694580078125, 195.03749084472656, 246.85888671875, 288.7082824707031, 348.61773681640625, 320.2000427246094, 374.10577392578125, 335.2696838378906, 740.977783203125, 1120.1212158203125]], "relative_norm": [[0.0029500878881663084, 0.06929938495159149, 0.00726969912648201, 0.3299228250980377, 0.17942048609256744, 0.1804026961326599, 0.23774130642414093, 0.29104211926460266, 0.36345815658569336, 0.29521578550338745, 0.3275122344493866, 0.37150561809539795, 0.670802891254425, 1.0952068567276]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81175231933594, 288.77423095703125, 348.5993347167969, 320.21826171875, 374.1238098144531, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.65830612182617, 55.04728698730469, 8.402240753173828, 374.5434265136719, 190.55694580078125, 195.03749084472656, 246.85888671875, 288.7082824707031, 348.61773681640625, 320.2000427246094, 374.10577392578125, 335.2696838378906, 740.977783203125, 1120.1212158203125]], "relative_norm": [[0.0029500878881663084, 0.06929938495159149, 0.00726969912648201, 0.3299228250980377, 0.17942048609256744, 0.1804026961326599, 0.23774130642414093, 0.29104211926460266, 0.36345815658569336, 0.29521578550338745, 0.3275122344493866, 0.37150561809539795, 0.670802891254425, 1.0952068567276]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.\n\nFact: The currency used in the country", "output_ids": [30109, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.66284942626953, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 320.630859375, 364.7038269042969, 328.0982971191406, 748.3380126953125, 1102.7435302734375]], "actual_norm": [[46.65830612182617, 54.65799331665039, 9.767363548278809, 371.5407409667969, 193.52261352539062, 196.79718017578125, 247.1049346923828, 292.3645324707031, 351.5674743652344, 320.59161376953125, 364.69024658203125, 328.0841064453125, 748.4432373046875, 1102.73974609375]], "relative_norm": [[0.0029500878881663084, 0.06983395665884018, 0.00845459196716547, 0.3282491862773895, 0.18198391795158386, 0.18236719071865082, 0.23856553435325623, 0.29383301734924316, 0.3657812476158142, 0.29571616649627686, 0.31984323263168335, 0.3625527620315552, 0.675083577632904, 1.0791889429092407]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 17672, 13, 19461, 32488, 37088, 304, 468, 63366, 892], "first_top_logits": [20.125, 19.0, 16.75, 16.75, 16.125, 16.0, 15.8125, 15.625, 15.5625, 15.5], "source_correct": true, "target_correct": false, "seconds": 0.44046554202213883, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-25/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-25", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Gothenburg", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Gothenburg is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.3816077709197998, "target_logp": -17.45842742919922, "log_odds": -17.07681965827942, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.008948225528001785, -0.37176382541656494, -0.0008956949459388852]}, "target": {"ids": [29691, 43479], "log_probs": [-16.696447372436523, -0.7619804739952087]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.7742004394531, 348.59930419921875, 320.21820068359375, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.61465072631836, 55.036338806152344, 8.378278732299805, 374.5426940917969, 190.54652404785156, 195.12216186523438, 246.89552307128906, 288.7300720214844, 348.5726623535156, 320.2130126953125, 374.10308837890625, 335.3114929199219, 741.0960083007812, 1120.2451171875]], "relative_norm": [[0.0029473276808857918, 0.06928560137748718, 0.007248966954648495, 0.3299221694469452, 0.1794106662273407, 0.18048101663589478, 0.23777659237384796, 0.2910640835762024, 0.36341115832328796, 0.2952277362346649, 0.32750988006591797, 0.3715519309043884, 0.6709099411964417, 1.0953279733657837]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.7742004394531, 348.59930419921875, 320.21820068359375, 374.123779296875, 335.2681884765625, 741.0166015625, 1120.0623779296875]], "actual_norm": [[46.61465072631836, 55.036338806152344, 8.378278732299805, 374.5426940917969, 190.54652404785156, 195.12216186523438, 246.89552307128906, 288.7300720214844, 348.5726623535156, 320.2130126953125, 374.10308837890625, 335.3114929199219, 741.0960083007812, 1120.2451171875]], "relative_norm": [[0.0029473276808857918, 0.06928560137748718, 0.007248966954648495, 0.3299221694469452, 0.1794106662273407, 0.18048101663589478, 0.23777659237384796, 0.2910640835762024, 0.36341115832328796, 0.2952277362346649, 0.32750988006591797, 0.3715519309043884, 0.6709099411964417, 1.0953279733657837]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.\n\nFact: The currency used in the country", "output_ids": [30109, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 5994, 3391, 10269, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[46.662841796875, 54.57722091674805, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81227111816406, 247.119873046875, 292.3907775878906, 351.5521240234375, 320.6308288574219, 364.70379638671875, 328.0982666015625, 748.3380126953125, 1102.7435302734375]], "actual_norm": [[46.61465072631836, 54.62799072265625, 9.724705696105957, 371.5484619140625, 193.52406311035156, 196.8594207763672, 247.15451049804688, 292.43212890625, 351.5833740234375, 320.6214294433594, 364.6882019042969, 328.1005554199219, 748.3424072265625, 1102.8621826171875]], "relative_norm": [[0.0029473276808857918, 0.069795623421669, 0.00841766782104969, 0.3282560110092163, 0.1819852739572525, 0.1824248731136322, 0.23861339688301086, 0.2939009666442871, 0.3657977879047394, 0.2957436442375183, 0.3198414444923401, 0.36257094144821167, 0.6749926805496216, 1.0793088674545288]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 23190, 5052, 63366, 110055, 84574, 13387, 86173, 4492, 17672], "first_top_logits": [27.125, 21.75, 19.875, 19.75, 19.5, 19.5, 19.375, 19.375, 18.625, 18.375], "source_correct": true, "target_correct": false, "seconds": 0.43358774995431304, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/readout", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 3636.0, "target_rank": 1106.0, "source_logit": 1.3046875, "target_logit": 2.890625, "top_ids": [15973, 11413, 45418, 8330, 6364, 7513, 4505, 3693, 49029, 38352, 7065, 3001], "top_tokens": ["currency", " currency", ".currency", " Australia", " English", " European", " Europe", " American", "ctica", "uenta", "ican", "ica"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 6466.0, "target_rank": 352.5, "source_logit": 0.369140625, "target_logit": 4.34375, "top_ids": [11413, 15973, 3001, 28453, 45418, 3693, 5088, 7065, 96694, 53661, 6364, 26321], "top_tokens": [" currency", "currency", "ica", " Currency", ".currency", " American", "urrency", "ican", "onis", "cla", " English", "Currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 359.0, "target_rank": 219.0, "source_logit": 16.25, "target_logit": 17.625, "top_ids": [11413, 35602, 32488, 85466, 7855, 8330, 12463, 6424, 7513, 13224, 26138, 13369], "top_tokens": [" currency", " currencies", " euros", " Euros", " British", " Australia", " colour", " UK", " European", " Britain", " colours", " Australian"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 367.0, "target_rank": 175.5, "source_logit": 13.375, "target_logit": 14.75, "top_ids": [11413, 13224, 7855, 35602, 7214, 47412, 3220, 74508, 32488, 65001, 9448, 17672], "top_tokens": [" currency", " Britain", " British", " currencies", " foreign", " foreigners", " money", "\u00bb.", " euros", "\"\u2014", " England", " euro"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 8871.5, "target_rank": 1985.0, "source_logit": -0.322265625, "target_logit": 1.8359375, "top_ids": [21251, 28453, 11413, 6364, 3731, 603, 7513, 45418, 3776, 15973, 90475, 55964], "top_tokens": ["onia", " Currency", " currency", " English", " Red", "one", " European", ".currency", " One", "currency", "(currency", "\u2011"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 6687.0, "target_rank": 2412.0, "source_logit": -0.09423828125, "target_logit": 1.4765625, "top_ids": [7179, 364, 11766, 21251, 11413, 2932, 6489, 4716, 28453, 467, 479, 18174], "top_tokens": [" International", " '", " Standard", "onia", " currency", " She", " international", " Bar", " Currency", " W", " G", "oret"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 2543.0, "target_rank": 762.5, "source_logit": 11.9375, "target_logit": 13.875, "top_ids": [11413, 35602, 3146, 13867, 7855, 29973, 6364, 28453, 5837, 7214, 32488, 9448], "top_tokens": [" currency", " currencies", " country", " county", " British", " licence", " English", " Currency", " countries", " foreign", " euros", " England"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 796.5, "target_rank": 98.0, "source_logit": 12.9375, "target_logit": 16.5, "top_ids": [7855, 11413, 7513, 35602, 17672, 37198, 6364, 103408, 5837, 6424, 32488, 4505], "top_tokens": [" British", " currency", " European", " currencies", " euro", " Commonwealth", " English", "\u6b27\u5143", " countries", " UK", " euros", " Europe"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 10527.5, "target_rank": 3189.0, "source_logit": -1.4375, "target_logit": 0.609375, "top_ids": [3693, 4787, 6364, 279, 89478, 4229, 5434, 271, 4102, 7855, 11, 220], "top_tokens": [" American", " North", " English", " the", "\u2026\n", " Ge", "\u2026\n\n", "\n\n", "\u00a0", " British", ",", " "]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 5294.5, "target_rank": 5939.5, "source_logit": -0.142578125, "target_logit": -0.33984375, "top_ids": [1852, 5938, 4787, 4102, 271, 6364, 3639, 3693, 479, 4882, 362, 1599], "top_tokens": [" same", " German", " North", "\u00a0", "\n\n", " English", " United", " American", " G", " South", " A", " X"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 8386.5, "target_rank": 1868.5, "source_logit": 7.375, "target_logit": 10.75, "top_ids": [11413, 35602, 104142, 103408, 28453, 112455, 32488, 85466, 113883, 26321, 110546, 90475], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " Currency", "\u82f1\u9551", " euros", " Euros", "\u672c\u56fd", "Currency", "\u82f1\u683c\u5170", "(currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 9057.5, "target_rank": 978.5, "source_logit": 1.765625, "target_logit": 6.34375, "top_ids": [104142, 103408, 112397, 116866, 106333, 106127, 101237, 110065, 113883, 109442, 112455, 105579], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u6b27\u76df", "\u7f8e\u5143", "\u6c47\u7387", "\u672c\u56fd", "\u8d22\u653f\u90e8", "\u82f1\u9551", "\u6fb3\u5927\u5229\u4e9a"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 5.0, "target_rank": 975.5, "source_logit": 21.125, "target_logit": 5.21875, "top_ids": [30109, 11413, 110055, 43680, 23190, 44621, 84574, 112455, 32488, 11192, 28453, 103408], "top_tokens": [" Swedish", " currency", "\u745e\u5178", " Danish", " Sweden", " Norwegian", " Scandinavian", "\u82f1\u9551", " euros", " dollars", " Currency", "\u6b27\u5143"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 3.0, "target_rank": 1566.0, "source_logit": 27.875, "target_logit": 4.0625, "top_ids": [30109, 110055, 23190, 43680, 84574, 44621, 63366, 11413, 52082, 96871, 34340, 57853], "top_tokens": [" Swedish", "\u745e\u5178", " Sweden", " Danish", " Scandinavian", " Norwegian", " Nordic", " currency", " Stockholm", "Sweden", " Denmark", " Finnish"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 18.0, "target_rank": 438.5, "source_logit": 31.375, "target_logit": 14.5625, "top_ids": [11413, 104142, 103408, 110055, 35602, 112455, 32488, 11192, 28453, 30109, 77186, 101406], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u745e\u5178", " currencies", "\u82f1\u9551", " euros", " dollars", " Currency", " Swedish", ".Currency", "\u8d27\u5e01"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 3.0, "target_rank": 598.0, "source_logit": 30.625, "target_logit": 8.75, "top_ids": [110055, 30109, 23190, 104142, 84574, 43680, 96871, 63366, 44621, 11413, 103408, 86173], "top_tokens": ["\u745e\u5178", " Swedish", " Sweden", "\u4eba\u6c11\u5e01", " Scandinavian", " Danish", "Sweden", " Nordic", " Norwegian", " currency", "\u6b27\u5143", " Svens"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 35.0, "target_rank": 9311.5, "source_logit": 17.875, "target_logit": 4.96875, "top_ids": [279, 30109, 87932, 3881, 43680, 19461, 96560, 17672, 3371, 220, 2598, 32488], "top_tokens": [" the", " Swedish", " kron", " known", " Danish", " Euro", " Kron", " euro", " den", " ", " called", " euros"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 18.5, "target_rank": 11776.0, "source_logit": 18.125, "target_logit": 2.296875, "top_ids": [30109, 87932, 96560, 5052, 43680, 110055, 84574, 44621, 23103, 17672, 19461, 63366], "top_tokens": [" Swedish", " kron", " Kron", " SE", " Danish", "\u745e\u5178", " Scandinavian", " Norwegian", " kr", " euro", " Euro", " Nordic"]}], "source_token": 23190, "target_token": 29121, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Mal", "mo", " is", " the"], "seconds": 1.16976920934394}
{"run_id": "eval-26/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.31312185525894165, "target_logp": -15.060928344726562, "log_odds": -14.74780648946762, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.04894530400633812, -0.2621406316757202, -0.0020359281916171312]}, "target": {"ids": [29691, 43479], "log_probs": [-14.230202674865723, -0.8307256102561951]}}, "score_trace": {"source": [], "target": []}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [], "first_top_ids": [30109, 87932, 96560, 5052, 43680, 110055, 84574, 44621, 23103, 17672], "first_top_logits": [28.375, 24.875, 23.625, 22.625, 21.875, 21.75, 21.125, 20.75, 20.25, 20.0], "source_correct": true, "target_correct": false, "seconds": 0.482767834328115, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.31312185525894165, "target_logp": -15.060928344726562, "log_odds": -14.74780648946762, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.04894530400633812, -0.2621406316757202, -0.0020359281916171312]}, "target": {"ids": [29691, 43479], "log_probs": [-14.230202674865723, -0.8307256102561951]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 87932, 96560, 5052, 43680, 110055, 84574, 44621, 23103, 17672], "first_top_logits": [28.375, 24.875, 23.625, 22.625, 21.875, 21.75, 21.125, 20.75, 20.25, 20.0], "source_correct": true, "target_correct": false, "seconds": 0.44119779113680124, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -35.170623779296875, "target_logp": -4.797318458557129, "log_odds": 30.373305320739746, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-34.76835250854492, -0.40100929141044617, -0.0012630588607862592]}, "target": {"ids": [29691, 43479], "log_probs": [-3.89577054977417, -0.9015477895736694]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 1.0991181135177612, 532.9072875976562, 888.7088012695312, 1267.4752197265625]], "actual_norm": [[46.685401916503906, 55.04582977294922, 8.306366920471191, 374.4843444824219, 190.5871124267578, 195.06292724609375, 246.84976196289062, 288.7644348144531, 348.5726013183594, 0.8415221571922302, 532.8834228515625, 888.6322021484375, 1267.376708984375]], "relative_norm": [[0.0029518010560423136, 0.06929755210876465, 0.007186748553067446, 0.3298707604408264, 0.17944888770580292, 0.180426225066185, 0.23773252964019775, 0.2910987138748169, 0.3634110987186432, 0.0006409251363947988, 0.5850452780723572, 0.8030065298080444, 1.2145379781723022]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 54.0387077331543, 9.601507186889648, 374.2162170410156, 190.38694763183594, 194.92945861816406, 248.83718872070312, 291.6499938964844, 348.9872741699219, 2.950425863265991, 534.1405639648438, 882.5951538085938, 1257.177490234375]], "actual_norm": [[46.685401916503906, 54.05760955810547, 9.671717643737793, 374.2154235839844, 190.48431396484375, 194.9679412841797, 248.8965606689453, 291.655029296875, 349.03125, 3.0252413749694824, 534.1867065429688, 882.5958251953125, 1257.1392822265625]], "relative_norm": [[0.0029518010560423136, 0.06822444498538971, 0.008349280804395676, 0.3300275206565857, 0.1794808954000473, 0.18035192787647247, 0.23990842700004578, 0.29387038946151733, 0.36294496059417725, 0.0022864167112857103, 0.5867260694503784, 0.7967187166213989, 1.2034755945205688]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Switzerland Franc.  What is the currency used in the country", "output_ids": [29121, 9694, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.37744140625, 843.42138671875, 1256.041015625]], "actual_norm": [[46.685401916503906, 54.61267852783203, 9.771011352539062, 371.4886779785156, 193.52008056640625, 196.80213928222656, 247.17041015625, 292.3385925292969, 351.5611267089844, 1.1786365509033203, 528.4403076171875, 843.37353515625, 1256.126953125]], "relative_norm": [[0.0029518010560423136, 0.06977605819702148, 0.008457749150693417, 0.3282032012939453, 0.18198153376579285, 0.18237179517745972, 0.23862874507904053, 0.29380694031715393, 0.36577463150024414, 0.0008946057059802115, 0.581096887588501, 0.7659152746200562, 1.1965036392211914]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29121, 29691, 108205, 6826, 13218, 71726, 19551, 16464, 26423, 220], "first_top_logits": [29.25, 25.25, 20.625, 15.8125, 14.0625, 13.5625, 13.3125, 13.1875, 12.9375, 12.6875], "source_correct": false, "target_correct": true, "seconds": 0.44123116601258516, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.245096683502197, "target_logp": -21.377593994140625, "log_odds": -15.132497310638428, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-5.883490562438965, -0.36070913076400757, -0.0008971242350526154]}, "target": {"ids": [29691, 43479], "log_probs": [-20.477113723754883, -0.9004799127578735]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 1.0991181135177612, 532.9072875976562, 888.7088012695312, 1267.4752197265625]], "actual_norm": [[46.64292526245117, 55.00143051147461, 8.3364896774292, 374.4998779296875, 190.60134887695312, 195.0970458984375, 246.79107666015625, 288.7442932128906, 348.5768127441406, 0.8674670457839966, 532.901123046875, 888.6853637695312, 1267.5718994140625]], "relative_norm": [[0.002949115354567766, 0.06924165785312653, 0.007212810683995485, 0.32988446950912476, 0.1794622838497162, 0.18045778572559357, 0.2376760095357895, 0.2910784184932709, 0.36341550946235657, 0.0006606854149140418, 0.585064709186554, 0.8030545711517334, 1.2147250175476074]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 54.0387077331543, 9.601507186889648, 374.2162170410156, 190.38694763183594, 194.92945861816406, 248.83718872070312, 291.6499938964844, 348.9872741699219, 2.950425863265991, 534.140625, 882.5951538085938, 1257.177490234375]], "actual_norm": [[46.64292526245117, 54.0261116027832, 9.642804145812988, 374.2344665527344, 190.39352416992188, 194.96914672851562, 248.89068603515625, 291.65203857421875, 348.9685363769531, 2.94211745262146, 534.2216796875, 882.5675659179688, 1257.300537109375]], "relative_norm": [[0.002949115354567766, 0.06818469613790512, 0.00832432135939598, 0.33004429936408997, 0.17939534783363342, 0.180353045463562, 0.23990276455879211, 0.2938673794269562, 0.36287975311279297, 0.0022235934156924486, 0.5867644548416138, 0.7966932058334351, 1.203629970550537]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Mal", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 8394], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 54.57722854614258, 9.721745491027832, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.3773803710938, 843.4213256835938, 1256.041015625]], "actual_norm": [[46.64292526245117, 54.57379913330078, 9.746484756469727, 371.4788513183594, 193.49276733398438, 196.79794311523438, 247.14505004882812, 292.3660583496094, 351.5832824707031, 1.2028989791870117, 528.3364868164062, 843.5601806640625, 1255.95263671875]], "relative_norm": [[0.002949115354567766, 0.0697263851761818, 0.008436519652605057, 0.3281945288181305, 0.18195584416389465, 0.18236790597438812, 0.23860426247119904, 0.29383453726768494, 0.3657976984977722, 0.0009130212711170316, 0.5809827446937561, 0.7660847902297974, 1.1963375806808472]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 30109, 6633, 220, 19551, 11413, 198, 13, 9338, 12236], "first_top_logits": [28.25, 22.5, 21.875, 21.5, 20.375, 20.125, 19.75, 19.0, 18.875, 18.875], "source_correct": false, "target_correct": false, "seconds": 0.4667335422709584, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.037660598754883, "target_logp": -33.64952850341797, "log_odds": -28.611867904663086, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-4.745001792907715, -0.2910557985305786, -0.001602914067916572]}, "target": {"ids": [29691, 43479], "log_probs": [-32.59248352050781, -1.0570449829101562]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.77423095703125, 348.59930419921875, 1.0991181135177612, 532.9072875976562, 888.7088012695312, 1267.4752197265625]], "actual_norm": [[46.659061431884766, 55.05573654174805, 8.30714225769043, 374.5028076171875, 190.60870361328125, 195.0790557861328, 246.83595275878906, 288.7991638183594, 348.57391357421875, 0.8422961831092834, 532.931884765625, 888.7186889648438, 1267.4718017578125]], "relative_norm": [[0.0029501356184482574, 0.0693100243806839, 0.007187419105321169, 0.3298870325088501, 0.17946921288967133, 0.18044114112854004, 0.23771922290325165, 0.29113373160362244, 0.363412469625473, 0.0006415146053768694, 0.5850985050201416, 0.803084671497345, 1.214629054069519]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 54.0387077331543, 9.601507186889648, 374.2162170410156, 190.38694763183594, 194.92945861816406, 248.83718872070312, 291.6499938964844, 348.9872741699219, 2.950425863265991, 534.1405639648438, 882.5951538085938, 1257.177490234375]], "actual_norm": [[46.659061431884766, 54.09165954589844, 9.663180351257324, 374.1479187011719, 190.38429260253906, 194.9579315185547, 248.8335723876953, 291.5959167480469, 348.9704284667969, 3.0265631675720215, 534.1434326171875, 882.5860595703125, 1257.2125244140625]], "relative_norm": [[0.0029501356184482574, 0.06826741993427277, 0.008341911248862743, 0.3299679756164551, 0.1793866604566574, 0.1803426742553711, 0.23984771966934204, 0.2938108444213867, 0.36288172006607056, 0.002287415787577629, 0.5866785049438477, 0.7967099547386169, 1.2035456895828247]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Sweden krona.\n\nFact: The currency used in the country", "output_ids": [23190, 87932, 64, 382, 17417, 25, 576, 11413, 1483, 304, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662845611572266, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.37744140625, 843.42138671875, 1256.041015625]], "actual_norm": [[46.659061431884766, 54.617027282714844, 9.773731231689453, 371.537109375, 193.53634643554688, 196.8165740966797, 247.10540771484375, 292.3741455078125, 351.58990478515625, 1.1740689277648926, 528.3838500976562, 843.4683837890625, 1256.1236572265625]], "relative_norm": [[0.0029501356184482574, 0.06978161633014679, 0.008460103534162045, 0.32824599742889404, 0.18199683725833893, 0.1823851764202118, 0.23856598138809204, 0.2938426733016968, 0.3658045828342438, 0.0008911387994885445, 0.5810348391532898, 0.7660014629364014, 1.196500539779663]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23190, 730, 595, 30109, 468, 773, 451, 1852, 220, 21665], "first_top_logits": [24.625, 22.125, 20.75, 20.25, 20.0, 19.0, 18.875, 18.375, 18.25, 18.0], "source_correct": true, "target_correct": false, "seconds": 0.45097925001755357, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.4517909288406372, "target_logp": -16.154260635375977, "log_odds": -15.70246970653534, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.012739231809973717, -0.43779268860816956, -0.001259010867215693]}, "target": {"ids": [29691, 43479], "log_probs": [-15.324758529663086, -0.8295027017593384]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277507781982422, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.8117218017578, 288.7742004394531, 348.59930419921875, 1.0991181135177612, 532.9072875976562, 888.708740234375, 1267.47509765625]], "actual_norm": [[46.60539627075195, 55.07643127441406, 8.313722610473633, 374.48187255859375, 190.56602478027344, 194.99717712402344, 246.8210906982422, 288.8201904296875, 348.5928039550781, 0.8867677450180054, 532.9403686523438, 888.7926025390625, 1267.376220703125]], "relative_norm": [[0.002946742344647646, 0.06933607161045074, 0.007193112745881081, 0.3298685848712921, 0.17942902445793152, 0.1803654134273529, 0.23770490288734436, 0.2911549210548401, 0.3634321689605713, 0.0006753853522241116, 0.5851078033447266, 0.8031514883041382, 1.214537501335144]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 54.03870391845703, 9.601507186889648, 374.2161865234375, 190.38694763183594, 194.92945861816406, 248.83717346191406, 291.6499938964844, 348.98724365234375, 2.950425863265991, 534.1405029296875, 882.5950927734375, 1257.177490234375]], "actual_norm": [[46.60539627075195, 54.102882385253906, 9.629101753234863, 374.2270812988281, 190.4459686279297, 194.86953735351562, 248.79879760742188, 291.66912841796875, 348.94476318359375, 3.0044875144958496, 534.1525268554688, 882.6453857421875, 1257.161376953125]], "relative_norm": [[0.002946742344647646, 0.06828158348798752, 0.008312491700053215, 0.33003780245780945, 0.17944477498531342, 0.18026089668273926, 0.23981419205665588, 0.29388460516929626, 0.3628550171852112, 0.0022707313764840364, 0.5866885185241699, 0.7967634797096252, 1.2034968137741089]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.50848388671875, 196.81228637695312, 247.119873046875, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.3773803710938, 843.4213256835938, 1256.0408935546875]], "actual_norm": [[46.60539627075195, 54.58580780029297, 9.756089210510254, 371.44281005859375, 193.45623779296875, 196.7270050048828, 247.15594482421875, 292.424560546875, 351.525146484375, 1.1839646100997925, 528.3633422851562, 843.349853515625, 1256.0687255859375]], "relative_norm": [[0.002946742344647646, 0.06974172592163086, 0.008444832637906075, 0.32816267013549805, 0.18192149698734283, 0.18230216205120087, 0.2386147826910019, 0.29389333724975586, 0.3657371997833252, 0.0008986497996374965, 0.5810122489929199, 0.765893816947937, 1.1964482069015503]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 17672, 19461, 110055, 4492, 30, 220, 32488, 84574, 5052], "first_top_logits": [25.625, 20.0, 19.375, 19.25, 19.0, 18.375, 18.0, 17.375, 17.0, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.44842583406716585, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.5944151878356934, "target_logp": -10.9667387008667, "log_odds": -10.372323513031006, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.42502084374427795, -0.16850106418132782, -0.0008933129138313234]}, "target": {"ids": [29691, 43479], "log_probs": [-10.1372709274292, -0.8294677138328552]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 55.00907516479492, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81175231933594, 288.77423095703125, 348.5993347167969, 1.0991181135177612, 532.9072875976562, 888.7088012695312, 1267.4752197265625]], "actual_norm": [[46.65830612182617, 55.04728698730469, 8.402240753173828, 374.5434265136719, 190.55694580078125, 195.03749084472656, 246.85888671875, 288.7082824707031, 348.61773681640625, 0.8802801370620728, 532.8092651367188, 888.6531372070312, 1267.275390625]], "relative_norm": [[0.0029500878881663084, 0.06929938495159149, 0.00726969912648201, 0.3299228250980377, 0.17942048609256744, 0.1804026961326599, 0.23774130642414093, 0.29104211926460266, 0.36345815658569336, 0.0006704442203044891, 0.584963858127594, 0.8030254244804382, 1.2144408226013184]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 54.0387077331543, 9.601508140563965, 374.2162170410156, 190.386962890625, 194.92947387695312, 248.8372039794922, 291.6499938964844, 348.9873046875, 2.950425863265991, 534.1405639648438, 882.5951538085938, 1257.177490234375]], "actual_norm": [[46.65830612182617, 54.09749221801758, 9.65207576751709, 374.23663330078125, 190.3970184326172, 194.90858459472656, 248.80023193359375, 291.6019592285156, 349.0326232910156, 3.000831365585327, 534.1502685546875, 882.6510009765625, 1257.0225830078125]], "relative_norm": [[0.0029500878881663084, 0.06827478110790253, 0.008332325145602226, 0.3300462067127228, 0.17939864099025726, 0.18029701709747314, 0.2398155778646469, 0.2938169240951538, 0.36294639110565186, 0.0022679681424051523, 0.5866860151290894, 0.7967685461044312, 1.2033638954162598]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.66284942626953, 54.57722473144531, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81228637695312, 247.11988830566406, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.37744140625, 843.4214477539062, 1256.0411376953125]], "actual_norm": [[46.65830612182617, 54.65799331665039, 9.767363548278809, 371.5407409667969, 193.52261352539062, 196.79718017578125, 247.1049346923828, 292.3645324707031, 351.5674743652344, 1.2317882776260376, 528.349853515625, 843.462646484375, 1256.099365234375]], "relative_norm": [[0.0029500878881663084, 0.06983395665884018, 0.00845459196716547, 0.3282491862773895, 0.18198391795158386, 0.18236719071865082, 0.23856553435325623, 0.29383301734924316, 0.3657812476158142, 0.0009349487954750657, 0.5809974074363708, 0.7659962177276611, 1.1964772939682007]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 17672, 13, 304, 892, 63366, 23190, 19461, 1112, 4492], "first_top_logits": [19.625, 17.125, 16.625, 16.25, 16.125, 16.125, 15.625, 15.5625, 15.5625, 15.5], "source_correct": true, "target_correct": false, "seconds": 0.46719908295199275, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-26/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-26", "split": "eval", "country": "Sweden", "currency": "Swedish krona", "aliases": ["krona", "SEK"], "city": "Malmo", "target_country": "Switzerland", "target_currency": "Swiss franc", "target_aliases": ["franc", "CHF"], "prompt": "Fact: The currency used in the country containing Malmo is the", "source_continuation": " Swedish krona", "target_continuation": " Swiss franc"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.35241058468818665, "target_logp": -18.741498947143555, "log_odds": -18.389088362455368, "token_scores": {"source": {"ids": [30109, 87932, 64], "log_probs": [-0.01565488427877426, -0.33575448393821716, -0.0010012142593041062]}, "target": {"ids": [29691, 43479], "log_probs": [-17.763900756835938, -0.9775985479354858]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 55.009071350097656, 8.277508735656738, 374.5055847167969, 190.56639099121094, 195.05657958984375, 246.81173706054688, 288.7742004394531, 348.59930419921875, 1.0991181135177612, 532.9072265625, 888.708740234375, 1267.47509765625]], "actual_norm": [[46.61465072631836, 55.036338806152344, 8.378278732299805, 374.5426940917969, 190.54652404785156, 195.12216186523438, 246.89552307128906, 288.7300720214844, 348.5726623535156, 0.8939608931541443, 532.8304443359375, 888.73828125, 1267.46435546875]], "relative_norm": [[0.0029473276808857918, 0.06928560137748718, 0.007248966954648495, 0.3299221694469452, 0.1794106662273407, 0.18048101663589478, 0.23777659237384796, 0.2910640835762024, 0.36341115832328796, 0.0006808637990616262, 0.5849871039390564, 0.8031023740768433, 1.214621901512146]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 54.03870391845703, 9.601507186889648, 374.2161865234375, 190.38694763183594, 194.92945861816406, 248.83717346191406, 291.6499938964844, 348.9872741699219, 2.950425863265991, 534.1405639648438, 882.5950927734375, 1257.177490234375]], "actual_norm": [[46.61465072631836, 54.10073471069336, 9.606748580932617, 374.2166442871094, 190.38279724121094, 194.96502685546875, 248.86614990234375, 291.6289367675781, 348.9198303222656, 3.070219039916992, 534.0822143554688, 882.5460205078125, 1257.0889892578125]], "relative_norm": [[0.0029473276808857918, 0.06827887147665024, 0.008293195627629757, 0.33002859354019165, 0.179385244846344, 0.1803492307662964, 0.23987911641597748, 0.2938441038131714, 0.3628290891647339, 0.0023204099852591753, 0.586611270904541, 0.7966737747192383, 1.2034274339675903]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swedish krona.  What is the currency used in the", "output_ids": [30109, 87932, 64, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8394, 6355, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[46.662841796875, 54.57722091674805, 9.721744537353516, 371.4899597167969, 193.5084991455078, 196.81227111816406, 247.119873046875, 292.3907775878906, 351.5521240234375, 1.3575661182403564, 528.3773803710938, 843.4213256835938, 1256.0408935546875]], "actual_norm": [[46.61465072631836, 54.62799072265625, 9.724705696105957, 371.5484619140625, 193.52406311035156, 196.8594207763672, 247.15451049804688, 292.43212890625, 351.5833740234375, 1.188256025314331, 528.3635864257812, 843.2909545898438, 1256.0577392578125]], "relative_norm": [[0.0029473276808857918, 0.069795623421669, 0.00841766782104969, 0.3282560110092163, 0.1819852739572525, 0.1824248731136322, 0.23861339688301086, 0.2939009666442871, 0.3657977879047394, 0.0009019070421345532, 0.5810125470161438, 0.7658402919769287, 1.1964377164840698]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [30109, 23190, 86173, 13387, 63366, 84574, 110055, 19551, 4492, 328], "first_top_logits": [26.125, 21.25, 19.125, 19.0, 19.0, 18.75, 18.625, 18.375, 18.125, 17.0], "source_correct": true, "target_correct": false, "seconds": 0.4699453338980675, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/readout", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 1449.0, "target_rank": 410.5, "source_logit": 2.203125, "target_logit": 3.78125, "top_ids": [8330, 38352, 7513, 15973, 45418, 11413, 4505, 13369, 3001, 43393, 7748, 47238], "top_tokens": [" Australia", "uenta", " European", "currency", ".currency", " currency", " Europe", " Australian", "ica", " Generally", " Indian", " Regulation"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 350.5, "target_rank": 1454.0, "source_logit": 4.0, "target_logit": 2.28125, "top_ids": [11413, 15973, 96694, 53661, 3001, 28453, 5088, 45418, 3693, 77186, 42189, 4787], "top_tokens": [" currency", "currency", "onis", "cla", "ica", " Currency", "urrency", ".currency", " American", ".Currency", "etz", " North"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 203.0, "target_rank": 495.0, "source_logit": 17.125, "target_logit": 15.1875, "top_ids": [11413, 35602, 7513, 8330, 6489, 3263, 4505, 9625, 7855, 6864, 24357, 5837], "top_tokens": [" currency", " currencies", " European", " Australia", " international", "\".", " Europe", " France", " British", " Canada", " overseas", " countries"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 199.5, "target_rank": 1026.5, "source_logit": 13.9375, "target_logit": 11.3125, "top_ids": [11413, 7214, 13224, 65001, 7855, 35602, 47412, 74508, 21130, 37198, 9625, 9448], "top_tokens": [" currency", " foreign", " Britain", "\"\u2014", " British", " currencies", " foreigners", "\u00bb.", " parliament", " Commonwealth", " France", " England"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 4706.0, "target_rank": 14243.5, "source_logit": 0.33984375, "target_logit": -1.4765625, "top_ids": [21251, 55964, 5737, 3776, 29921, 28453, 7513, 19833, 3731, 56537, 9127, 603], "top_tokens": ["onia", "\u2011", " Group", " One", "\u2010", " Currency", " European", " Amb", " Red", " _______,", " Rock", "one"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 4459.5, "target_rank": 19949.5, "source_logit": 0.083984375, "target_logit": -2.515625, "top_ids": [7179, 364, 479, 11766, 2932, 6489, 11413, 356, 467, 3639, 362, 21251], "top_tokens": [" International", " '", " G", " Standard", " She", " international", " currency", " C", " W", " United", " A", "onia"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1803.5, "target_rank": 4308.0, "source_logit": 12.8125, "target_logit": 11.375, "top_ids": [11413, 35602, 13867, 3146, 5313, 5837, 30176, 3283, 43895, 7513, 28453, 7214], "top_tokens": [" currency", " currencies", " county", " country", " national", " countries", " counties", " city", " tariffs", " European", " Currency", " foreign"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 176.5, "target_rank": 966.0, "source_logit": 14.9375, "target_logit": 12.3125, "top_ids": [37198, 7513, 11413, 7855, 6489, 5837, 7214, 6364, 35602, 5313, 94936, 17672], "top_tokens": [" Commonwealth", " European", " currency", " British", " international", " countries", " foreign", " English", " currencies", " national", " denomination", " euro"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3345.0, "target_rank": 15133.5, "source_logit": 0.482421875, "target_logit": -2.140625, "top_ids": [3693, 89478, 4787, 279, 4229, 2240, 2598, 271, 5434, 1083, 220, 6364], "top_tokens": [" American", "\u2026\n", " North", " the", " Ge", " None", " called", "\n\n", "\u2026\n\n", " also", " ", " English"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 7009.5, "target_rank": 8925.5, "source_logit": -0.71484375, "target_logit": -1.15625, "top_ids": [1852, 5938, 3693, 3639, 271, 4102, 4787, 479, 362, 7179, 220, 6364], "top_tokens": [" same", " German", " American", " United", "\n\n", "\u00a0", " North", " G", " A", " International", " ", " English"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 3131.5, "target_rank": 5162.0, "source_logit": 9.9375, "target_logit": 8.8125, "top_ids": [11413, 35602, 103408, 104142, 28453, 32488, 113883, 115160, 85466, 26321, 113596, 90475], "top_tokens": [" currency", " currencies", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " Currency", " euros", "\u672c\u56fd", "\u5176\u4ed6\u56fd\u5bb6", " Euros", "Currency", "\u56fd\u7c4d", "(currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 793.0, "target_rank": 4782.0, "source_logit": 6.46875, "target_logit": 2.984375, "top_ids": [104142, 103408, 101406, 101237, 112397, 116866, 106333, 106127, 110065, 109442, 113883, 116588], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u8d27\u5e01", "\u7f8e\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u6b27\u76df", "\u6c47\u7387", "\u8d22\u653f\u90e8", "\u672c\u56fd", "\u6210\u5458\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 20.5, "target_rank": 14375.0, "source_logit": 17.0, "target_logit": -0.1943359375, "top_ids": [11413, 32488, 103408, 85466, 11192, 17672, 104142, 35602, 17692, 2598, 19551, 112455], "top_tokens": [" currency", " euros", "\u6b27\u5143", " Euros", " dollars", " euro", "\u4eba\u6c11\u5e01", " currencies", " dollar", " called", " USD", "\u82f1\u9551"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 12.0, "target_rank": 7861.5, "source_logit": 18.875, "target_logit": 0.62890625, "top_ids": [29691, 11413, 103408, 17672, 104142, 32488, 85466, 17692, 5938, 108205, 19551, 29121], "top_tokens": [" Swiss", " currency", "\u6b27\u5143", " euro", "\u4eba\u6c11\u5e01", " euros", " Euros", " dollar", " German", "\u745e\u58eb", " USD", " Switzerland"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 32.5, "target_rank": 9440.0, "source_logit": 27.125, "target_logit": 4.65625, "top_ids": [103408, 11413, 32488, 104142, 35602, 11192, 85466, 112455, 28453, 17672, 17692, 101237], "top_tokens": ["\u6b27\u5143", " currency", " euros", "\u4eba\u6c11\u5e01", " currencies", " dollars", " Euros", "\u82f1\u9551", " Currency", " euro", " dollar", "\u7f8e\u5143"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 21.0, "target_rank": 9035.5, "source_logit": 22.625, "target_logit": 1.59375, "top_ids": [103408, 104142, 32488, 11413, 35602, 17672, 101406, 108205, 101237, 112455, 28453, 85466], "top_tokens": ["\u6b27\u5143", "\u4eba\u6c11\u5e01", " euros", " currency", " currencies", " euro", "\u8d27\u5e01", "\u745e\u58eb", "\u7f8e\u5143", "\u82f1\u9551", " Currency", " Euros"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 15.5, "target_rank": 49680.0, "source_logit": 19.125, "target_logit": -1.25, "top_ids": [279, 29691, 3881, 2598, 19461, 43479, 9694, 85466, 17672, 2176, 220, 4492], "top_tokens": [" the", " Swiss", " known", " called", " Euro", " franc", " Franc", " Euros", " euro", " both", " ", " Sw"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 11.5, "target_rank": 50386.5, "source_logit": 18.875, "target_logit": -2.796875, "top_ids": [29691, 19461, 17672, 9694, 5938, 72366, 108205, 7513, 6826, 1532, 29121, 4492], "top_tokens": [" Swiss", " Euro", " euro", " Franc", " German", " Canton", "\u745e\u58eb", " European", " CH", " New", " Switzerland", " Sw"]}], "source_token": 29121, "target_token": 27696, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Zurich", " is", " the"], "seconds": 1.1547141666524112}
{"run_id": "eval-27/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.9770269989967346, "target_logp": -26.294843673706055, "log_odds": -25.31781667470932, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.0009317824151366949, -0.9760951995849609]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-25.06340980529785, -1.2238595485687256, -0.007573936600238085]}}, "score_trace": {"source": [], "target": []}, "output": " Swiss Franc.  In the country containing the city of Geneva", "output_ids": [29691, 9694, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [], "first_top_ids": [29691, 19461, 17672, 9694, 5938, 72366, 108205, 6826, 7513, 1532], "first_top_logits": [30.5, 22.875, 21.875, 21.0, 20.75, 19.5, 19.375, 19.125, 19.125, 19.0], "source_correct": true, "target_correct": false, "seconds": 0.4908877913840115, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.9770269989967346, "target_logp": -26.294843673706055, "log_odds": -25.31781667470932, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.0009317824151366949, -0.9760951995849609]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-25.06340980529785, -1.2238595485687256, -0.007573936600238085]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swiss Franc.  In the country containing the city of Geneva", "output_ids": [29691, 9694, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29691, 19461, 17672, 9694, 5938, 72366, 108205, 6826, 7513, 1532], "first_top_logits": [30.5, 22.875, 21.875, 21.0, 20.75, 19.5, 19.375, 19.125, 19.125, 19.0], "source_correct": true, "target_correct": false, "seconds": 0.4312859163619578, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -41.360782623291016, "target_logp": -4.9147162437438965, "log_odds": 36.44606637954712, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-40.460758209228516, -0.9000232219696045]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-3.773256778717041, -1.1373459100723267, -0.0041139028035104275]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 962.478759765625, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8363037109375, 84.82217407226562, 0.10680145025253296, 21.927961349487305, 27.163238525390625, 89.1976089477539, 45.59315490722656, 153.36575317382812, 200.0296173095703, 962.5009155273438, 1053.218505859375, 1063.6468505859375]], "relative_norm": [[0.021550225093960762, 0.10837331414222717, 9.244692046195269e-05, 0.019372938200831413, 0.025543643161654472, 0.08265727013349533, 0.04401755705475807, 0.1541360765695572, 0.20811675488948822, 0.9842650890350342, 0.9666195511817932, 1.0447081327438354]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 960.7280883789062, 1059.0965576171875, 1060.042236328125]], "actual_norm": [[340.8363037109375, 89.52792358398438, 3.8516156673431396, 22.638601303100586, 24.670557022094727, 88.37446594238281, 47.302669525146484, 152.18470764160156, 198.3274383544922, 960.7393798828125, 1059.006103515625, 1060.0712890625]], "relative_norm": [[0.021550225093960762, 0.11299043893814087, 0.00332497525960207, 0.0199654009193182, 0.023245450109243393, 0.08174936473369598, 0.0455944798886776, 0.153340682387352, 0.20623351633548737, 0.981719434261322, 0.9733093976974487, 1.0456899404525757]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysia Ringgit.   What would be the best way to", "output_ids": [27696, 21525, 12882, 13, 256, 3555, 1035, 387, 279, 1850, 1616, 311], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 962.478759765625, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8363037109375, 84.82217407226562, 0.10680145025253296, 21.927961349487305, 27.163238525390625, 89.1976089477539, 45.59315490722656, 153.36575317382812, 200.0296173095703, 962.5009155273438, 1053.218505859375, 1063.6468505859375]], "relative_norm": [[0.021550225093960762, 0.10837331414222717, 9.244692046195269e-05, 0.019372938200831413, 0.025543643161654472, 0.08265727013349533, 0.04401755705475807, 0.1541360765695572, 0.20811675488948822, 0.9842650890350342, 0.9666195511817932, 1.0447081327438354]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27696, 65431, 109123, 79539, 86333, 58085, 10058, 8394, 21525, 364], "first_top_logits": [33.5, 29.75, 21.125, 20.25, 20.0, 19.375, 19.25, 19.0, 18.875, 18.625], "source_correct": false, "target_correct": true, "seconds": 0.4507155418395996, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -6.574193477630615, "target_logp": -24.33763885498047, "log_odds": -17.763445377349854, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-5.260084629058838, -1.3141087293624878]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-23.10191535949707, -1.2239972352981567, -0.011726848781108856]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.0413818359375, 962.4788208007812, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8644104003906, 84.820068359375, 0.09754247963428497, 21.938854217529297, 27.168344497680664, 89.13612365722656, 45.563655853271484, 153.34429931640625, 200.04031372070312, 962.5493774414062, 1053.3695068359375, 1063.685546875]], "relative_norm": [[0.02155200205743313, 0.10837062448263168, 8.443239494226873e-05, 0.01938256062567234, 0.02554844506084919, 0.08260029554367065, 0.04398907721042633, 0.1541145145893097, 0.20812788605690002, 0.9843146800994873, 0.966758131980896, 1.0447460412979126]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 960.7280883789062, 1059.0965576171875, 1060.042236328125]], "actual_norm": [[340.8644104003906, 89.5265884399414, 3.8996994495391846, 22.60464859008789, 24.667142868041992, 88.38436126708984, 47.28586196899414, 152.2686767578125, 198.31838989257812, 960.6448974609375, 1058.997802734375, 1060.0072021484375]], "relative_norm": [[0.02155200205743313, 0.11298875510692596, 0.0033664843067526817, 0.01993545889854431, 0.02324223332107067, 0.0817585214972496, 0.045578278601169586, 0.15342529118061066, 0.20622411370277405, 0.9816228747367859, 0.9733017683029175, 1.0456267595291138]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency used in the country containing", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.0413818359375, 962.4788208007812, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8644104003906, 84.820068359375, 0.09754247963428497, 21.938854217529297, 27.168344497680664, 89.13612365722656, 45.563655853271484, 153.34429931640625, 200.04031372070312, 962.5493774414062, 1053.3695068359375, 1063.685546875]], "relative_norm": [[0.02155200205743313, 0.10837062448263168, 8.443239494226873e-05, 0.01938256062567234, 0.02554844506084919, 0.08260029554367065, 0.04398907721042633, 0.1541145145893097, 0.20812788605690002, 0.9843146800994873, 0.966758131980896, 1.0447460412979126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 29691, 85466, 17672, 220, 19461, 7513, 6633, 13, 198], "first_top_logits": [27.25, 22.0, 21.125, 19.25, 19.125, 19.125, 18.75, 18.375, 18.25, 18.25], "source_correct": false, "target_correct": false, "seconds": 0.44838708313181996, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.8140130043029785, "target_logp": -39.916561126708984, "log_odds": -36.102548122406006, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-2.5892536640167236, -1.2247594594955444]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-38.5894775390625, -1.313722848892212, -0.013357671909034252]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 962.478759765625, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8729248046875, 84.8004150390625, 0.10643819719552994, 21.90896987915039, 27.17937469482422, 89.19586181640625, 45.61627197265625, 153.32887268066406, 200.05880737304688, 962.4214477539062, 1053.2823486328125, 1063.5113525390625]], "relative_norm": [[0.021552540361881256, 0.10834551602602005, 9.213248995365575e-05, 0.01935615949332714, 0.025558816269040108, 0.0826556533575058, 0.04403987526893616, 0.1540990173816681, 0.20814712345600128, 0.9841838479042053, 0.9666781425476074, 1.0445749759674072]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 960.7280883789062, 1059.0965576171875, 1060.042236328125]], "actual_norm": [[340.8729248046875, 89.55194091796875, 3.8497424125671387, 22.645954132080078, 24.68327522277832, 88.38581848144531, 47.2961540222168, 152.19256591796875, 198.36199951171875, 960.7100219726562, 1059.171142578125, 1060.052490234375]], "relative_norm": [[0.021552540361881256, 0.11302074790000916, 0.00332335801795125, 0.01997188664972782, 0.023257434368133545, 0.08175986260175705, 0.04558819904923439, 0.15334859490394592, 0.20626945793628693, 0.981689453125, 0.9734610319137573, 1.0456714630126953]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Switzerland Franc.  In the country that contains the city of", "output_ids": [29121, 9694, 13, 220, 758, 279, 3146, 429, 5610, 279, 3283, 315], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 962.478759765625, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.8729248046875, 84.8004150390625, 0.10643819719552994, 21.90896987915039, 27.17937469482422, 89.19586181640625, 45.61627197265625, 153.32887268066406, 200.05880737304688, 962.4214477539062, 1053.2823486328125, 1063.5113525390625]], "relative_norm": [[0.021552540361881256, 0.10834551602602005, 9.213248995365575e-05, 0.01935615949332714, 0.025558816269040108, 0.0826556533575058, 0.04403987526893616, 0.1540990173816681, 0.20814712345600128, 0.9841838479042053, 0.9666781425476074, 1.0445749759674072]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29121, 29691, 6826, 108205, 71726, 2149, 5938, 14168, 5329, 17226], "first_top_logits": [19.875, 17.375, 14.1875, 13.9375, 13.3125, 13.1875, 11.375, 11.3125, 11.1875, 10.9375], "source_correct": true, "target_correct": false, "seconds": 0.44117699982598424, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.0516549348831177, "target_logp": -22.89954948425293, "log_odds": -21.847894549369812, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.07594982534646988, -0.9757051467895508]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-21.483434677124023, -1.4068522453308105, -0.009261621162295341]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 84.82992553710938, 0.20234881341457367, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.3499298095703, 200.04139709472656, 962.4786987304688, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.4182434082031, 84.86463928222656, 0.0921284481883049, 21.94706153869629, 27.180980682373047, 89.21617889404297, 45.58259582519531, 153.37530517578125, 200.07200622558594, 962.4591064453125, 1053.4039306640625, 1063.6358642578125]], "relative_norm": [[0.021523792296648026, 0.10842756927013397, 7.974603067850694e-05, 0.019389811903238297, 0.025560326874256134, 0.08267448097467422, 0.04400736093521118, 0.15414567291736603, 0.20816084742546082, 0.9842223525047302, 0.966789722442627, 1.0446972846984863]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 89.51358032226562, 3.785341262817383, 22.59021759033203, 24.6097354888916, 88.35150146484375, 47.26787185668945, 152.20840454101562, 198.3301239013672, 960.72802734375, 1059.0965576171875, 1060.0421142578125]], "actual_norm": [[340.4182434082031, 89.5168685913086, 3.8910768032073975, 22.64153289794922, 24.61688804626465, 88.31593322753906, 47.25165557861328, 152.2783660888672, 198.3250732421875, 960.7003173828125, 1059.1168212890625, 1059.9757080078125]], "relative_norm": [[0.021523792296648026, 0.11297648400068283, 0.0033590407110750675, 0.01996798627078533, 0.02319488115608692, 0.08169522136449814, 0.0455453060567379, 0.15343505144119263, 0.20623105764389038, 0.9816794991493225, 0.9734111428260803, 1.045595645904541]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swiss Franc.  The currency used in the country containing Geneva", "output_ids": [29691, 9694, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 84.82992553710938, 0.20234881341457367, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.3499298095703, 200.04139709472656, 962.4786987304688, 1053.2889404296875, 1063.630126953125]], "actual_norm": [[340.4182434082031, 84.86463928222656, 0.0921284481883049, 21.94706153869629, 27.180980682373047, 89.21617889404297, 45.58259582519531, 153.37530517578125, 200.07200622558594, 962.4591064453125, 1053.4039306640625, 1063.6358642578125]], "relative_norm": [[0.021523792296648026, 0.10842756927013397, 7.974603067850694e-05, 0.019389811903238297, 0.025560326874256134, 0.08267448097467422, 0.04400736093521118, 0.15414567291736603, 0.20816084742546082, 0.9842223525047302, 0.966789722442627, 1.0446972846984863]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29691, 17672, 19461, 8585, 43479, 9694, 4492, 33315, 55734, 5938], "first_top_logits": [24.375, 21.125, 20.75, 19.25, 17.75, 17.5, 17.375, 16.875, 16.75, 16.5], "source_correct": true, "target_correct": false, "seconds": 0.40819370932877064, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -0.8940324187278748, "target_logp": -24.65668487548828, "log_odds": -23.762652456760406, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.06670690327882767, -0.8273255228996277]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-23.33292579650879, -1.313663125038147, -0.010095132514834404]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.8299331665039, 0.20234884321689606, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.04139709472656, 962.4788208007812, 1053.2890625, 1063.6302490234375]], "actual_norm": [[340.8020935058594, 84.82697296142578, 0.10364914685487747, 21.934471130371094, 27.20722007751465, 89.15409088134766, 45.53080368041992, 153.38108825683594, 199.99749755859375, 962.4464111328125, 1053.35791015625, 1063.6588134765625]], "relative_norm": [[0.021548062562942505, 0.10837944597005844, 8.971829811343923e-05, 0.01937868818640709, 0.025585001334547997, 0.08261694759130478, 0.043957360088825226, 0.15415148437023163, 0.20808333158493042, 0.9842093586921692, 0.966747522354126, 1.0447198152542114]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341739654541, 22.590219497680664, 24.609739303588867, 88.35150909423828, 47.26787567138672, 152.20843505859375, 198.3301544189453, 960.7280883789062, 1059.0965576171875, 1060.042236328125]], "actual_norm": [[340.8020935058594, 89.53717041015625, 3.9032506942749023, 22.619298934936523, 24.674766540527344, 88.35215759277344, 47.22201156616211, 152.2064208984375, 198.43536376953125, 960.7534790039062, 1059.0848388671875, 1059.9368896484375]], "relative_norm": [[0.021548062562942505, 0.11300210654735565, 0.0033695499878376722, 0.019948378205299377, 0.023249417543411255, 0.0817287266254425, 0.0455167330801487, 0.15336255729198456, 0.20634575188159943, 0.9817338585853577, 0.973381757736206, 1.0455573797225952]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swiss Franc.  In the country containing the city of Geneva", "output_ids": [29691, 9694, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.8299331665039, 0.20234884321689606, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.04139709472656, 962.4788208007812, 1053.2890625, 1063.6302490234375]], "actual_norm": [[340.8020935058594, 84.82697296142578, 0.10364914685487747, 21.934471130371094, 27.20722007751465, 89.15409088134766, 45.53080368041992, 153.38108825683594, 199.99749755859375, 962.4464111328125, 1053.35791015625, 1063.6588134765625]], "relative_norm": [[0.021548062562942505, 0.10837944597005844, 8.971829811343923e-05, 0.01937868818640709, 0.025585001334547997, 0.08261694759130478, 0.043957360088825226, 0.15415148437023163, 0.20808333158493042, 0.9842093586921692, 0.966747522354126, 1.0447198152542114]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29691, 19461, 17672, 13, 12137, 892, 264, 85466, 29121, 5938], "first_top_logits": [22.75, 19.125, 18.125, 17.625, 17.375, 16.875, 16.75, 16.375, 16.375, 16.25], "source_correct": true, "target_correct": false, "seconds": 0.4244385417550802, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-27/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-27", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Zurich", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Zurich is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2327616214752197, "target_logp": -21.770877838134766, "log_odds": -20.538116216659546, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.008099921979010105, -1.2246617078781128]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-20.538837432861328, -1.2241185903549194, -0.007921826094388962]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.34994506835938, 200.04139709472656, 962.4786987304688, 1053.2889404296875, 1063.6300048828125]], "actual_norm": [[340.4613952636719, 84.83588409423828, 0.11337139457464218, 22.04853057861328, 27.158584594726562, 89.12861633300781, 45.596160888671875, 153.3192596435547, 200.08055114746094, 962.4544677734375, 1053.370361328125, 1063.6029052734375]], "relative_norm": [[0.02152652107179165, 0.10839083045721054, 9.813383803702891e-05, 0.019479459151625633, 0.02553926594555378, 0.08259333670139313, 0.044020459055900574, 0.15408934652805328, 0.20816974341869354, 0.9842175841331482, 0.9667589068412781, 1.0446648597717285]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.6097354888916, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.3301239013672, 960.72802734375, 1059.0965576171875, 1060.042236328125]], "actual_norm": [[340.4613952636719, 89.49616241455078, 3.8805625438690186, 22.750123977661133, 24.677881240844727, 88.39191436767578, 47.239688873291016, 152.18350219726562, 198.33323669433594, 960.6893310546875, 1059.0565185546875, 1059.90283203125]], "relative_norm": [[0.02152652107179165, 0.11295035481452942, 0.003349964041262865, 0.02006375603377819, 0.023252351209521294, 0.08176550269126892, 0.04553377255797386, 0.1533394604921341, 0.20623955130577087, 0.9816682934761047, 0.9733557105064392, 1.045523762702942]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swiss Franc.  In the country containing the city of Geneva", "output_ids": [29691, 9694, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 71726, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.34994506835938, 200.04139709472656, 962.4786987304688, 1053.2889404296875, 1063.6300048828125]], "actual_norm": [[340.4613952636719, 84.83588409423828, 0.11337139457464218, 22.04853057861328, 27.158584594726562, 89.12861633300781, 45.596160888671875, 153.3192596435547, 200.08055114746094, 962.4544677734375, 1053.370361328125, 1063.6029052734375]], "relative_norm": [[0.02152652107179165, 0.10839083045721054, 9.813383803702891e-05, 0.019479459151625633, 0.02553926594555378, 0.08259333670139313, 0.044020459055900574, 0.15408934652805328, 0.20816974341869354, 0.9842175841331482, 0.9667589068412781, 1.0446648597717285]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29691, 55734, 17672, 29121, 4492, 5938, 19461, 17692, 3639, 220], "first_top_logits": [25.75, 19.75, 19.125, 18.625, 18.625, 18.5, 18.125, 17.875, 17.25, 16.875], "source_correct": true, "target_correct": false, "seconds": 0.41622154181823134, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/readout", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 649.5, "target_rank": 354.5, "source_logit": 2.984375, "target_logit": 3.671875, "top_ids": [7513, 8330, 4505, 38352, 47238, 45418, 43393, 3001, 13369, 3693, 11413, 7748], "top_tokens": [" European", " Australia", " Europe", "uenta", " Regulation", ".currency", " Generally", "ica", " Australian", " American", " currency", " Indian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 335.5, "target_rank": 1456.5, "source_logit": 4.03125, "target_logit": 2.21875, "top_ids": [11413, 15973, 96694, 5088, 28453, 42189, 3001, 45418, 3693, 53661, 77186, 18174], "top_tokens": [" currency", "currency", "onis", "urrency", " Currency", "etz", "ica", ".currency", " American", "cla", ".Currency", "oret"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 129.0, "target_rank": 1136.5, "source_logit": 17.875, "target_logit": 13.0, "top_ids": [11413, 3263, 35602, 7513, 12463, 32488, 22956, 85466, 1189, 26138, 330, 36911], "top_tokens": [" currency", "\".", " currencies", " European", " colour", " euros", "\".\n", " Euros", ".\"", " colours", " \"", ".\"."]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 384.0, "target_rank": 903.5, "source_logit": 13.125, "target_logit": 11.8125, "top_ids": [11413, 35602, 7214, 13224, 3220, 32488, 7855, 17672, 74508, 78253, 21130, 15973], "top_tokens": [" currency", " currencies", " foreign", " Britain", " money", " euros", " British", " euro", "\u00bb.", " treasury", " parliament", "currency"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 6189.0, "target_rank": 18294.0, "source_logit": 0.1474609375, "target_logit": -1.6875, "top_ids": [29921, 21251, 603, 55964, 3776, 5737, 1717, 56537, 2926, 7513, 3731, 1111], "top_tokens": ["\u2010", "onia", "one", "\u2011", " One", " Group", "less", " _______,", " Dr", " European", " Red", "ule"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 5473.0, "target_rank": 16657.0, "source_logit": -0.1650390625, "target_logit": -2.078125, "top_ids": [7179, 479, 364, 11413, 11766, 2932, 356, 64, 21251, 18174, 362, 28453], "top_tokens": [" International", " G", " '", " currency", " Standard", " She", " C", "a", "onia", "oret", " A", " Currency"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1702.0, "target_rank": 13236.0, "source_logit": 12.75, "target_logit": 9.1875, "top_ids": [11413, 13867, 35602, 30176, 5313, 3283, 6290, 29973, 3146, 56818, 32679, 26414], "top_tokens": [" currency", " county", " currencies", " counties", " national", " city", " town", " licence", " country", " municipality", " citizenship", " municipal"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 244.5, "target_rank": 868.5, "source_logit": 13.9375, "target_logit": 12.125, "top_ids": [11413, 35602, 37198, 7513, 7855, 17672, 5313, 94936, 6364, 7214, 13867, 6489], "top_tokens": [" currency", " currencies", " Commonwealth", " European", " British", " euro", " national", " denomination", " English", " foreign", " county", " international"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3713.0, "target_rank": 22843.5, "source_logit": 0.609375, "target_logit": -2.625, "top_ids": [3693, 4102, 279, 111012, 271, 11, 89478, 5434, 1083, 2598, 5023, 30743], "top_tokens": [" American", "\u00a0", " the", "\u4e16\u754c\u4e0a\u6700", "\n\n", ",", "\u2026\n", "\u2026\n\n", " also", " called", " currently", " ____"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 8319.0, "target_rank": 7791.5, "source_logit": -0.94921875, "target_logit": -0.83984375, "top_ids": [1852, 3693, 4102, 5938, 479, 271, 3639, 7179, 4787, 2274, 11413, 362], "top_tokens": [" same", " American", "\u00a0", " German", " G", "\n\n", " United", " International", " North", " US", " currency", " A"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 3315.5, "target_rank": 13770.0, "source_logit": 10.1875, "target_logit": 6.96875, "top_ids": [11413, 35602, 103408, 104142, 28453, 32488, 85466, 26321, 101406, 113596, 17672, 15973], "top_tokens": [" currency", " currencies", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " Currency", " euros", " Euros", "Currency", "\u8d27\u5e01", "\u56fd\u7c4d", " euro", "currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 1307.0, "target_rank": 2408.0, "source_logit": 6.03125, "target_logit": 4.875, "top_ids": [104142, 103408, 106333, 110065, 101237, 109442, 101406, 35602, 112397, 112455, 116866, 106127], "top_tokens": ["\u4eba\u6c11\u5e01", "\u6b27\u5143", "\u5916\u6c47", "\u6c47\u7387", "\u7f8e\u5143", "\u8d22\u653f\u90e8", "\u8d27\u5e01", " currencies", "\u56fd\u5bb6\u6807\u51c6", "\u82f1\u9551", "\u76f4\u8f96\u5e02", "\u6b27\u76df"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 87.5, "target_rank": 5703.5, "source_logit": 11.625, "target_logit": 1.890625, "top_ids": [11413, 103408, 32488, 2598, 11192, 28453, 104142, 35602, 85466, 112455, 17692, 17672], "top_tokens": [" currency", "\u6b27\u5143", " euros", " called", " dollars", " Currency", "\u4eba\u6c11\u5e01", " currencies", " Euros", "\u82f1\u9551", " dollar", " euro"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 42.5, "target_rank": 1949.0, "source_logit": 13.9375, "target_logit": 3.453125, "top_ids": [11413, 103408, 104142, 17672, 32488, 17692, 35602, 85466, 19551, 28453, 11192, 29691], "top_tokens": [" currency", "\u6b27\u5143", "\u4eba\u6c11\u5e01", " euro", " euros", " dollar", " currencies", " Euros", " USD", " Currency", " dollars", " Swiss"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 80.5, "target_rank": 3008.0, "source_logit": 22.125, "target_logit": 9.5625, "top_ids": [11413, 103408, 35602, 32488, 28453, 104142, 11192, 112455, 85466, 101406, 17672, 17692], "top_tokens": [" currency", "\u6b27\u5143", " currencies", " euros", " Currency", "\u4eba\u6c11\u5e01", " dollars", "\u82f1\u9551", " Euros", "\u8d27\u5e01", " euro", " dollar"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 47.0, "target_rank": 1731.0, "source_logit": 19.25, "target_logit": 6.90625, "top_ids": [103408, 104142, 35602, 32488, 11413, 112455, 17672, 101237, 101406, 28453, 85466, 34710], "top_tokens": ["\u6b27\u5143", "\u4eba\u6c11\u5e01", " currencies", " euros", " currency", "\u82f1\u9551", " euro", "\u7f8e\u5143", "\u8d27\u5e01", " Currency", " Euros", "_currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 231.5, "target_rank": 9546.0, "source_logit": 12.6875, "target_logit": 5.03125, "top_ids": [279, 3881, 2598, 29691, 19461, 17672, 7855, 1083, 103408, 85466, 43479, 32488], "top_tokens": [" the", " known", " called", " Swiss", " Euro", " euro", " British", " also", "\u6b27\u5143", " Euros", " franc", " euros"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 74.5, "target_rank": 13608.5, "source_logit": 15.0625, "target_logit": 2.75, "top_ids": [19461, 17672, 29691, 7513, 103408, 8585, 9812, 7855, 37088, 85466, 43479, 9694], "top_tokens": [" Euro", " euro", " Swiss", " European", "\u6b27\u5143", " French", " EU", " British", " EUR", " Euros", " franc", " Franc"]}], "source_token": 29121, "target_token": 27696, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Basel", " is", " the"], "seconds": 1.1739957919344306}
{"run_id": "eval-28/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.118849277496338, "target_logp": -19.41320037841797, "log_odds": -16.29435110092163, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-2.1427409648895264, -0.9761083126068115]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-18.272428512573242, -1.1372255086898804, -0.003546616993844509]}}, "score_trace": {"source": [], "target": []}, "output": " Euro.  In the country containing the city of Basel,", "output_ids": [19461, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 89458, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [], "first_top_ids": [19461, 17672, 29691, 7513, 103408, 8585, 9812, 7855, 85466, 37088], "first_top_logits": [29.625, 29.5, 28.25, 25.375, 23.375, 23.0, 21.625, 21.5, 21.375, 21.375], "source_correct": false, "target_correct": false, "seconds": 0.49936999985948205, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.118849277496338, "target_logp": -19.41320037841797, "log_odds": -16.29435110092163, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-2.1427409648895264, -0.9761083126068115]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-18.272428512573242, -1.1372255086898804, -0.003546616993844509]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Euro.  In the country containing the city of Basel,", "output_ids": [19461, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 89458, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [19461, 17672, 29691, 7513, 103408, 8585, 9812, 7855, 85466, 37088], "first_top_logits": [29.625, 29.5, 28.25, 25.375, 23.375, 23.0, 21.625, 21.5, 21.375, 21.375], "source_correct": false, "target_correct": false, "seconds": 0.4509447501040995, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -33.64497756958008, "target_logp": -1.8326419591903687, "log_odds": 31.81233561038971, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-32.816322326660156, -0.828653872013092]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-0.6932948231697083, -1.1371926069259644, -0.002154411980882287]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8363037109375, 84.82217407226562, 0.10680145025253296, 21.927961349487305, 27.163238525390625, 89.1976089477539, 45.59315490722656, 153.36575317382812, 200.0296173095703, 449.03875732421875, 604.1499633789062, 599.2088012695312]], "relative_norm": [[0.021550225093960762, 0.10837331414222717, 9.244692046195269e-05, 0.019372938200831413, 0.025543643161654472, 0.08265727013349533, 0.04401755705475807, 0.1541360765695572, 0.20811675488948822, 0.4276620149612427, 0.5510201454162598, 0.5915439128875732]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 443.12103271484375, 581.5172119140625, 594.82275390625]], "actual_norm": [[340.8363037109375, 89.52792358398438, 3.8516156673431396, 22.638601303100586, 24.670557022094727, 88.37446594238281, 47.302669525146484, 152.18470764160156, 198.3274383544922, 443.1759033203125, 581.54833984375, 594.7826538085938]], "relative_norm": [[0.021550225093960762, 0.11299043893814087, 0.00332497525960207, 0.0199654009193182, 0.023245450109243393, 0.08174936473369598, 0.0455944798886776, 0.153340682387352, 0.20623351633548737, 0.42077523469924927, 0.534514307975769, 0.5865711569786072]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysia Ringgit.  What is the currency used in the", "output_ids": [27696, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8363037109375, 84.82217407226562, 0.10680145025253296, 21.927961349487305, 27.163238525390625, 89.1976089477539, 45.59315490722656, 153.36575317382812, 200.0296173095703, 449.03875732421875, 604.1499633789062, 599.2088012695312]], "relative_norm": [[0.021550225093960762, 0.10837331414222717, 9.244692046195269e-05, 0.019372938200831413, 0.025543643161654472, 0.08265727013349533, 0.04401755705475807, 0.1541360765695572, 0.20811675488948822, 0.4276620149612427, 0.5510201454162598, 0.5915439128875732]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27696, 65431, 17692, 109123, 10058, 8394, 21525, 2205, 79539, 7748], "first_top_logits": [31.75, 31.625, 22.0, 21.625, 21.25, 21.125, 20.875, 20.5, 20.125, 19.75], "source_correct": false, "target_correct": true, "seconds": 0.4344411250203848, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.570978164672852, "target_logp": -17.255319595336914, "log_odds": -11.684341430664062, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-4.346309661865234, -1.2246686220169067]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-16.19693946838379, -1.0540728569030762, -0.0043071601539850235]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.0413818359375, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8644104003906, 84.820068359375, 0.09754247963428497, 21.938854217529297, 27.168344497680664, 89.13612365722656, 45.563655853271484, 153.34429931640625, 200.04031372070312, 449.0538330078125, 604.2830810546875, 599.3675537109375]], "relative_norm": [[0.02155200205743313, 0.10837062448263168, 8.443239494226873e-05, 0.01938256062567234, 0.02554844506084919, 0.08260029554367065, 0.04398907721042633, 0.1541145145893097, 0.20812788605690002, 0.42767637968063354, 0.5511415600776672, 0.5917006731033325]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 443.12103271484375, 581.5172119140625, 594.8226928710938]], "actual_norm": [[340.8644104003906, 89.5265884399414, 3.8996994495391846, 22.60464859008789, 24.667142868041992, 88.38436126708984, 47.28586196899414, 152.2686767578125, 198.31838989257812, 443.0910339355469, 581.5059204101562, 594.843017578125]], "relative_norm": [[0.02155200205743313, 0.11298875510692596, 0.0033664843067526817, 0.01993545889854431, 0.02324223332107067, 0.0817585214972496, 0.045578278601169586, 0.15342529118061066, 0.20622411370277405, 0.42069464921951294, 0.5344752669334412, 0.5866307020187378]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  In the country containing the city of Basel,", "output_ids": [17672, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 89458, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.0413818359375, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8644104003906, 84.820068359375, 0.09754247963428497, 21.938854217529297, 27.168344497680664, 89.13612365722656, 45.563655853271484, 153.34429931640625, 200.04031372070312, 449.0538330078125, 604.2830810546875, 599.3675537109375]], "relative_norm": [[0.02155200205743313, 0.10837062448263168, 8.443239494226873e-05, 0.01938256062567234, 0.02554844506084919, 0.08260029554367065, 0.04398907721042633, 0.1541145145893097, 0.20812788605690002, 0.42767637968063354, 0.5511415600776672, 0.5917006731033325]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 1965, 7513, 29691, 85466, 103408, 32488, 8585, 7855], "first_top_logits": [25.875, 25.875, 25.625, 23.75, 22.625, 22.25, 21.0, 20.5, 20.125, 19.75], "source_correct": false, "target_correct": false, "seconds": 0.4387869997881353, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.2264846563339233, "target_logp": -35.75727081298828, "log_odds": -34.53078615665436, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-0.08748269826173782, -1.1390019655227661]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-34.525028228759766, -1.2237497568130493, -0.008490181528031826]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8729248046875, 84.8004150390625, 0.10643819719552994, 21.90896987915039, 27.17937469482422, 89.19586181640625, 45.61627197265625, 153.32887268066406, 200.05880737304688, 449.0728759765625, 604.0269775390625, 599.2034301757812]], "relative_norm": [[0.021552540361881256, 0.10834551602602005, 9.213248995365575e-05, 0.01935615949332714, 0.025558816269040108, 0.0826556533575058, 0.04403987526893616, 0.1540990173816681, 0.20814712345600128, 0.42769452929496765, 0.5509080290794373, 0.5915386080741882]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.609737396240234, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.33013916015625, 443.12103271484375, 581.5172119140625, 594.82275390625]], "actual_norm": [[340.8729248046875, 89.55194091796875, 3.8497424125671387, 22.645954132080078, 24.68327522277832, 88.38581848144531, 47.2961540222168, 152.19256591796875, 198.36199951171875, 443.1318359375, 581.584228515625, 594.8735961914062]], "relative_norm": [[0.021552540361881256, 0.11302074790000916, 0.00332335801795125, 0.01997188664972782, 0.023257434368133545, 0.08175986260175705, 0.04558819904923439, 0.15334859490394592, 0.20626945793628693, 0.42073339223861694, 0.5345472693443298, 0.5866608619689941]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Swiss Franc.  The currency used in the country containing Geneva", "output_ids": [29691, 9694, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 44245], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044006347656, 153.34994506835938, 200.04139709472656, 449.0592956542969, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8729248046875, 84.8004150390625, 0.10643819719552994, 21.90896987915039, 27.17937469482422, 89.19586181640625, 45.61627197265625, 153.32887268066406, 200.05880737304688, 449.0728759765625, 604.0269775390625, 599.2034301757812]], "relative_norm": [[0.021552540361881256, 0.10834551602602005, 9.213248995365575e-05, 0.01935615949332714, 0.025558816269040108, 0.0826556533575058, 0.04403987526893616, 0.1540990173816681, 0.20814712345600128, 0.42769452929496765, 0.5509080290794373, 0.5915386080741882]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [29691, 29121, 6826, 8585, 5938, 44245, 108205, 9694, 43479, 220], "first_top_logits": [24.0, 21.5, 18.625, 17.5, 17.25, 16.125, 15.9375, 15.625, 15.5, 14.625], "source_correct": true, "target_correct": false, "seconds": 0.49210175033658743, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.7479891777038574, "target_logp": -20.74881935119629, "log_odds": -17.00083017349243, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-2.7717740535736084, -0.9762150049209595]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-19.520164489746094, -1.2239124774932861, -0.004741733893752098]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 84.82992553710938, 0.20234881341457367, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.3499298095703, 200.04139709472656, 449.05926513671875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.4182434082031, 84.86463928222656, 0.0921284481883049, 21.94706153869629, 27.180980682373047, 89.21617889404297, 45.58259582519531, 153.37530517578125, 200.07200622558594, 449.1522216796875, 604.2229614257812, 599.3035888671875]], "relative_norm": [[0.021523792296648026, 0.10842756927013397, 7.974603067850694e-05, 0.019389811903238297, 0.025560326874256134, 0.08267448097467422, 0.04400736093521118, 0.15414567291736603, 0.20816084742546082, 0.42777007818222046, 0.5510867238044739, 0.5916374921798706]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 89.51358032226562, 3.785341262817383, 22.59021759033203, 24.6097354888916, 88.35150146484375, 47.26787185668945, 152.20840454101562, 198.3301239013672, 443.1210021972656, 581.5172119140625, 594.8226928710938]], "actual_norm": [[340.4182434082031, 89.5168685913086, 3.8910768032073975, 22.64153289794922, 24.61688804626465, 88.31593322753906, 47.25165557861328, 152.2783660888672, 198.3250732421875, 443.1924743652344, 581.4437866210938, 594.8660278320312]], "relative_norm": [[0.021523792296648026, 0.11297648400068283, 0.0033590407110750675, 0.01996798627078533, 0.02319488115608692, 0.08169522136449814, 0.0455453060567379, 0.15343505144119263, 0.20623105764389038, 0.4207909405231476, 0.5344181656837463, 0.5866534113883972]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  The currency used in the country containing the city", "output_ids": [17672, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482, 279, 3283], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.85723876953125, 84.82992553710938, 0.20234881341457367, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.3499298095703, 200.04139709472656, 449.05926513671875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.4182434082031, 84.86463928222656, 0.0921284481883049, 21.94706153869629, 27.180980682373047, 89.21617889404297, 45.58259582519531, 153.37530517578125, 200.07200622558594, 449.1522216796875, 604.2229614257812, 599.3035888671875]], "relative_norm": [[0.021523792296648026, 0.10842756927013397, 7.974603067850694e-05, 0.019389811903238297, 0.025560326874256134, 0.08267448097467422, 0.04400736093521118, 0.15414567291736603, 0.20816084742546082, 0.42777007818222046, 0.5510867238044739, 0.5916374921798706]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 29691, 8585, 103408, 7513, 468, 9694, 43479, 37088], "first_top_logits": [27.75, 27.5, 25.625, 22.125, 21.625, 21.5, 21.375, 19.625, 19.5, 19.375], "source_correct": false, "target_correct": false, "seconds": 0.44048641668632627, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -4.991033554077148, "target_logp": -21.39073944091797, "log_odds": -16.39970588684082, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-4.091064453125, -0.8999693393707275]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-20.248437881469727, -1.137226939201355, -0.005075190216302872]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.8299331665039, 0.20234884321689606, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.04139709472656, 449.059326171875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8020935058594, 84.82697296142578, 0.10364914685487747, 21.934471130371094, 27.20722007751465, 89.15409088134766, 45.53080368041992, 153.38108825683594, 199.99749755859375, 449.0471496582031, 604.11865234375, 599.2120971679688]], "relative_norm": [[0.021548062562942505, 0.10837944597005844, 8.971829811343923e-05, 0.01937868818640709, 0.025585001334547997, 0.08261694759130478, 0.043957360088825226, 0.15415148437023163, 0.20808333158493042, 0.42767003178596497, 0.5509915947914124, 0.5915471911430359]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341739654541, 22.590219497680664, 24.609739303588867, 88.35150909423828, 47.26787567138672, 152.20843505859375, 198.3301544189453, 443.12103271484375, 581.5172119140625, 594.82275390625]], "actual_norm": [[340.8020935058594, 89.53717041015625, 3.9032506942749023, 22.619298934936523, 24.674766540527344, 88.35215759277344, 47.22201156616211, 152.2064208984375, 198.43536376953125, 443.1219177246094, 581.432373046875, 594.8117065429688]], "relative_norm": [[0.021548062562942505, 0.11300210654735565, 0.0033695499878376722, 0.019948378205299377, 0.023249417543411255, 0.0817287266254425, 0.0455167330801487, 0.15336255729198456, 0.20634575188159943, 0.42072397470474243, 0.5344076752662659, 0.5865998268127441]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  In the country containing the city of Basel,", "output_ids": [17672, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 89458, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.8299331665039, 0.20234884321689606, 21.885133743286133, 27.15860939025879, 89.13945007324219, 45.56044387817383, 153.34994506835938, 200.04139709472656, 449.059326171875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.8020935058594, 84.82697296142578, 0.10364914685487747, 21.934471130371094, 27.20722007751465, 89.15409088134766, 45.53080368041992, 153.38108825683594, 199.99749755859375, 449.0471496582031, 604.11865234375, 599.2120971679688]], "relative_norm": [[0.021548062562942505, 0.10837944597005844, 8.971829811343923e-05, 0.01937868818640709, 0.025585001334547997, 0.08261694759130478, 0.043957360088825226, 0.15415148437023163, 0.20808333158493042, 0.42767003178596497, 0.5509915947914124, 0.5915471911430359]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 19461, 29691, 7513, 37088, 85466, 468, 9812, 8585, 43479], "first_top_logits": [27.75, 26.75, 24.0, 23.0, 21.5, 20.625, 20.625, 20.5, 20.375, 19.875], "source_correct": false, "target_correct": false, "seconds": 0.43138716742396355, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-28/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-28", "split": "eval", "country": "Switzerland", "currency": "Swiss franc", "aliases": ["franc", "CHF"], "city": "Basel", "target_country": "Malaysia", "target_currency": "Malaysian ringgit", "target_aliases": ["ringgit", "MYR"], "prompt": "Fact: The currency used in the country containing Basel is the", "source_continuation": " Swiss franc", "target_continuation": " Malaysian ringgit"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.233736038208008, "target_logp": -18.393678665161133, "log_odds": -16.159942626953125, "token_scores": {"source": {"ids": [29691, 43479], "log_probs": [-1.1780829429626465, -1.0556530952453613]}, "target": {"ids": [65431, 10058, 12882], "log_probs": [-17.334707260131836, -1.0542595386505127, -0.004710885230451822]}}, "score_trace": {"source": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.34994506835938, 200.04139709472656, 449.05926513671875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.4613952636719, 84.83588409423828, 0.11337139457464218, 22.04853057861328, 27.158584594726562, 89.12861633300781, 45.596160888671875, 153.3192596435547, 200.08055114746094, 449.0453796386719, 604.2020263671875, 599.3173217773438]], "relative_norm": [[0.02152652107179165, 0.10839083045721054, 9.813383803702891e-05, 0.019479459151625633, 0.02553926594555378, 0.08259333670139313, 0.044020459055900574, 0.15408934652805328, 0.20816974341869354, 0.42766833305358887, 0.5510676503181458, 0.5916510820388794]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 89.51358032226562, 3.785341501235962, 22.590219497680664, 24.6097354888916, 88.35150909423828, 47.26787567138672, 152.2084197998047, 198.3301239013672, 443.12103271484375, 581.5172119140625, 594.8226928710938]], "actual_norm": [[340.4613952636719, 89.49616241455078, 3.8805625438690186, 22.750123977661133, 24.677881240844727, 88.39191436767578, 47.239688873291016, 152.18350219726562, 198.33323669433594, 443.0738220214844, 581.4910888671875, 594.8413696289062]], "relative_norm": [[0.02152652107179165, 0.11295035481452942, 0.003349964041262865, 0.02006375603377819, 0.023252351209521294, 0.08176550269126892, 0.04553377255797386, 0.1533394604921341, 0.20623955130577087, 0.4206782877445221, 0.5344616770744324, 0.5866290926933289]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " euro.  In the country containing the city of Basel,", "output_ids": [17672, 13, 220, 758, 279, 3146, 8482, 279, 3283, 315, 89458, 11], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 89458, 374, 279], "generation_trace": [{"sequence_length": 12, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], "requested_norm": [[340.8572692871094, 84.82992553710938, 0.20234882831573486, 21.885133743286133, 27.158607482910156, 89.13944244384766, 45.5604362487793, 153.34994506835938, 200.04139709472656, 449.05926513671875, 604.1484985351562, 599.2301025390625]], "actual_norm": [[340.4613952636719, 84.83588409423828, 0.11337139457464218, 22.04853057861328, 27.158584594726562, 89.12861633300781, 45.596160888671875, 153.3192596435547, 200.08055114746094, 449.0453796386719, 604.2020263671875, 599.3173217773438]], "relative_norm": [[0.02152652107179165, 0.10839083045721054, 9.813383803702891e-05, 0.019479459151625633, 0.02553926594555378, 0.08259333670139313, 0.044020459055900574, 0.15408934652805328, 0.20816974341869354, 0.42766833305358887, 0.5510676503181458, 0.5916510820388794]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [17672, 29691, 19461, 7513, 5938, 7855, 17692, 43479, 11888, 9694], "first_top_logits": [27.25, 26.875, 26.625, 21.875, 20.625, 19.875, 19.625, 19.375, 19.375, 19.25], "source_correct": false, "target_correct": false, "seconds": 0.4335075421258807, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/readout", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 199.5, "target_rank": 129.5, "source_logit": 4.25, "target_logit": 4.65625, "top_ids": [8330, 38352, 45418, 7256, 7065, 1111, 6364, 58085, 7748, 31491, 47238, 43393], "top_tokens": [" Australia", "uenta", ".currency", "ige", "ican", "ule", " English", "aysia", " Indian", "uesto", " Regulation", " Generally"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 429.0, "target_rank": 2871.0, "source_logit": 3.796875, "target_logit": 1.375, "top_ids": [11413, 15973, 96694, 28453, 3001, 53661, 5088, 3693, 7065, 45418, 21295, 4787], "top_tokens": [" currency", "currency", "onis", " Currency", "ica", "cla", "urrency", " American", "ican", ".currency", "ollar", " North"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 65.5, "target_rank": 45.5, "source_logit": 18.375, "target_logit": 19.25, "top_ids": [12463, 26138, 38664, 91984, 11413, 69384, 29973, 3263, 40862, 13369, 8330, 22956], "top_tokens": [" colour", " colours", " recognised", " licences", " currency", " authorised", " licence", "\".", " Colour", " Australian", " Australia", "\".\n"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 147.5, "target_rank": 193.5, "source_logit": 14.5, "target_logit": 14.0, "top_ids": [11413, 13224, 7855, 7214, 37198, 21130, 3220, 13369, 8330, 65001, 35602, 47412], "top_tokens": [" currency", " Britain", " British", " foreign", " Commonwealth", " parliament", " money", " Australian", " Australia", "\"\u2014", " currencies", " foreigners"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 9107.0, "target_rank": 8152.5, "source_logit": -0.83203125, "target_logit": -0.65234375, "top_ids": [603, 3776, 3731, 55964, 21251, 5737, 7748, 29921, 12751, 11753, 19833, 4229], "top_tokens": ["one", " One", " Red", "\u2011", "onia", " Group", " Indian", "\u2010", " Roman", " Tw", " Amb", " Ge"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 7040.0, "target_rank": 8552.5, "source_logit": -0.890625, "target_logit": -1.2265625, "top_ids": [364, 7179, 64, 479, 11766, 198, 362, 11413, 4716, 467, 356, 264], "top_tokens": [" '", " International", "a", " G", " Standard", "\n", " A", " currency", " Bar", " W", " C", " a"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 949.0, "target_rank": 1458.0, "source_logit": 12.8125, "target_logit": 12.25, "top_ids": [11413, 29973, 13867, 91984, 12463, 35602, 3146, 5313, 30176, 53275, 37198, 6290], "top_tokens": [" currency", " licence", " county", " licences", " colour", " currencies", " country", " national", " counties", " Licence", " Commonwealth", " town"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 208.5, "target_rank": 630.0, "source_logit": 14.0625, "target_logit": 12.5, "top_ids": [37198, 11413, 7855, 94936, 7214, 6364, 105492, 13867, 5313, 35602, 16847, 3639], "top_tokens": [" Commonwealth", " currency", " British", " denomination", " foreign", " English", "\u4e2d\u534e\u4eba\u6c11", " county", " national", " currencies", " province", " United"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 12232.0, "target_rank": 7249.0, "source_logit": -1.6328125, "target_logit": -0.69140625, "top_ids": [89478, 4102, 5434, 279, 4787, 3693, 4229, 11, 1083, 46427, 220, 271], "top_tokens": ["\u2026\n", "\u00a0", "\u2026\n\n", " the", " North", " American", " Ge", ",", " also", "/is", " ", "\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 2869.0, "target_rank": 8889.0, "source_logit": 0.90234375, "target_logit": -0.98046875, "top_ids": [1852, 271, 3693, 4102, 3639, 7179, 5938, 4882, 220, 4787, 5055, 2701], "top_tokens": [" same", "\n\n", " American", "\u00a0", " United", " International", " German", " South", " ", " North", " National", " following"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 585.5, "target_rank": 1157.5, "source_logit": 14.1875, "target_logit": 12.875, "top_ids": [11413, 35602, 28453, 104142, 103408, 32488, 91984, 85466, 113883, 90475, 6424, 26321], "top_tokens": [" currency", " currencies", " Currency", "\u4eba\u6c11\u5e01", "\u6b27\u5143", " euros", " licences", " Euros", "\u672c\u56fd", "(currency", " UK", "Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 261.5, "target_rank": 1825.5, "source_logit": 9.5, "target_logit": 5.5625, "top_ids": [104142, 116866, 106333, 101237, 112397, 109442, 105492, 113883, 101406, 103408, 110065, 105394], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u5916\u6c47", "\u7f8e\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u8d22\u653f\u90e8", "\u4e2d\u534e\u4eba\u6c11", "\u672c\u56fd", "\u8d27\u5e01", "\u6b27\u5143", "\u6c47\u7387", "\u65b0\u52a0\u5761"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 35.0, "target_rank": 137.0, "source_logit": 14.5625, "target_logit": 10.0, "top_ids": [11413, 104142, 11192, 17692, 7855, 28453, 112455, 35602, 80189, 42364, 19551, 15973], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " dollar", " British", " Currency", "\u82f1\u9551", " currencies", " Dollars", " Dollar", " USD", "currency"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 12.5, "target_rank": 108.0, "source_logit": 18.125, "target_logit": 10.5, "top_ids": [104142, 11413, 65431, 17692, 7855, 28453, 3639, 11192, 42364, 35602, 109123, 27696], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Malaysian", " dollar", " British", " Currency", " United", " dollars", " Dollar", " currencies", "\u9a6c\u6765\u897f\u4e9a", " Malaysia"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 34.0, "target_rank": 120.5, "source_logit": 27.625, "target_logit": 20.625, "top_ids": [104142, 11413, 112455, 35602, 11192, 28453, 17692, 80189, 101237, 101406, 77186, 42364], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", "\u82f1\u9551", " currencies", " dollars", " Currency", " dollar", " Dollars", "\u7f8e\u5143", "\u8d27\u5e01", ".Currency", " Dollar"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 16.5, "target_rank": 91.5, "source_logit": 26.0, "target_logit": 16.25, "top_ids": [104142, 65431, 11413, 28453, 35602, 112455, 101237, 17692, 101406, 77186, 110065, 109123], "top_tokens": ["\u4eba\u6c11\u5e01", " Malaysian", " currency", " Currency", " currencies", "\u82f1\u9551", "\u7f8e\u5143", " dollar", "\u8d27\u5e01", ".Currency", "\u6c47\u7387", "\u9a6c\u6765\u897f\u4e9a"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 8.0, "target_rank": 174.0, "source_logit": 20.75, "target_logit": 13.8125, "top_ids": [279, 65431, 7855, 3881, 20539, 10058, 21525, 27696, 220, 3639, 109123, 1532], "top_tokens": [" the", " Malaysian", " British", " known", " Singapore", " ring", " Ring", " Malaysia", " ", " United", "\u9a6c\u6765\u897f\u4e9a", " New"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 78.0, "source_logit": 23.0, "target_logit": 14.1875, "top_ids": [65431, 20539, 21525, 27696, 10058, 1532, 17692, 2274, 109123, 7855, 431, 3639], "top_tokens": [" Malaysian", " Singapore", " Ring", " Malaysia", " ring", " New", " dollar", " US", "\u9a6c\u6765\u897f\u4e9a", " British", " R", " United"]}], "source_token": 27696, "target_token": 23968, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Pen", "ang", " is", " the"], "seconds": 1.1509979590773582}
{"run_id": "eval-29/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.2842705249786377, "target_logp": -12.823267936706543, "log_odds": -10.538997411727905, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.04454784095287323, -2.2379631996154785, -0.001759529928676784]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-10.669548034667969, -2.1292338371276855, -0.023491401225328445, -0.000993354362435639]}}, "score_trace": {"source": [], "target": []}, "output": " Malaysian Ringgit.  The currency used in the country containing", "output_ids": [65431, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [], "first_top_ids": [65431, 20539, 21525, 27696, 10058, 17692, 1532, 2274, 109123, 7855], "first_top_logits": [28.25, 24.75, 23.5, 23.0, 22.125, 21.125, 21.125, 20.875, 20.625, 20.25], "source_correct": true, "target_correct": false, "seconds": 0.510073458775878, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.2842705249786377, "target_logp": -12.823267936706543, "log_odds": -10.538997411727905, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.04454784095287323, -2.2379631996154785, -0.001759529928676784]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-10.669548034667969, -2.1292338371276855, -0.023491401225328445, -0.000993354362435639]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  The currency used in the country containing", "output_ids": [65431, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 21525, 27696, 10058, 17692, 1532, 2274, 109123, 7855], "first_top_logits": [28.25, 24.75, 23.5, 23.0, 22.125, 21.125, 21.125, 20.875, 20.625, 20.25], "source_correct": true, "target_correct": false, "seconds": 0.48045587493106723, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -34.144065856933594, "target_logp": -4.642721176147461, "log_odds": 29.501344680786133, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-32.12454605102539, -2.0178067684173584, -0.0017134759109467268]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-2.8120455741882324, -1.8066496849060059, -0.022901218384504318, -0.0011250602547079325]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.71397399902344, 114.66758728027344, 5.452879905700684, 134.96002197265625, 128.23233032226562, 47.52134323120117, 39.822601318359375, 28.868017196655273, 106.67507934570312, 45.923465728759766, 743.649658203125, 373.4059143066406, 559.7268676757812]], "relative_norm": [[0.010857044719159603, 0.14435575902462006, 0.004717884119600058, 0.11888178437948227, 0.12073821574449539, 0.043955542147159576, 0.03835177794098854, 0.029101377353072166, 0.11121616512537003, 0.0353153720498085, 0.7607064247131348, 0.3405304551124573, 0.5418252944946289]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.71397399902344, 114.66758728027344, 5.452879905700684, 134.96002197265625, 128.23233032226562, 47.52134323120117, 39.822601318359375, 28.868017196655273, 106.67507934570312, 45.923465728759766, 743.649658203125, 373.4059143066406, 559.7268676757812]], "relative_norm": [[0.010857044719159603, 0.14435575902462006, 0.004717884119600058, 0.11888178437948227, 0.12073821574449539, 0.043955542147159576, 0.03835177794098854, 0.029101377353072166, 0.11121616512537003, 0.0353153720498085, 0.7607064247131348, 0.3405304551124573, 0.5418252944946289]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesia Rupiah.  What is the name of the", "output_ids": [23968, 431, 454, 19310, 13, 220, 3555, 374, 279, 829, 315, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 110.60438537597656, 3.079007148742676, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.760746002197266, 29.888771057128906, 109.08887481689453, 43.76106643676758, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.71397399902344, 110.58658599853516, 3.1973142623901367, 130.1429901123047, 128.72093200683594, 41.8874397277832, 41.737518310546875, 29.905061721801758, 109.12342834472656, 43.77987289428711, 741.336181640625, 368.5910339355469, 557.0198974609375]], "relative_norm": [[0.010857044719159603, 0.1412912905216217, 0.0027675828896462917, 0.1149788647890091, 0.12104600667953491, 0.03881607949733734, 0.04029516130685806, 0.030055267736315727, 0.11353525519371033, 0.03363517299294472, 0.7586504220962524, 0.337157666683197, 0.5394023656845093]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23968, 58829, 2263, 111348, 7748, 84378, 1257, 20263, 2274, 1425], "first_top_logits": [37.0, 34.0, 27.0, 26.25, 23.75, 23.625, 23.125, 22.375, 22.25, 22.125], "source_correct": false, "target_correct": true, "seconds": 0.46590533293783665, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.013291120529175, "target_logp": -9.930719375610352, "log_odds": -6.917428255081177, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.9933593273162842, -2.0178449153900146, -0.002086844528093934]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.993359088897705, -1.9122825860977173, -0.024073906242847443, -0.00100419158115983]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.75006103515625, 114.65499114990234, 5.465886116027832, 134.89073181152344, 128.28697204589844, 47.506553649902344, 39.78873825073242, 28.91241455078125, 106.74829864501953, 45.96055603027344, 743.7216186523438, 373.3681945800781, 559.7249755859375]], "relative_norm": [[0.010859326459467411, 0.1443399041891098, 0.004729137290269136, 0.11882074922323227, 0.12078966945409775, 0.043941862881183624, 0.038319166749715805, 0.029146132990717888, 0.11129250377416611, 0.035343896597623825, 0.7607800364494324, 0.3404960632324219, 0.5418234467506409]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.75006103515625, 114.65499114990234, 5.465886116027832, 134.89073181152344, 128.28697204589844, 47.506553649902344, 39.78873825073242, 28.91241455078125, 106.74829864501953, 45.96055603027344, 743.7216186523438, 373.3681945800781, 559.7249755859375]], "relative_norm": [[0.010859326459467411, 0.1443399041891098, 0.004729137290269136, 0.11882074922323227, 0.12078966945409775, 0.043941862881183624, 0.038319166749715805, 0.029146132990717888, 0.11129250377416611, 0.035343896597623825, 0.7607800364494324, 0.3404960632324219, 0.5418234467506409]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency used in the country containing", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 110.6043930053711, 3.079007148742676, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.760746002197266, 29.888771057128906, 109.08887481689453, 43.76106262207031, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.75006103515625, 110.65412139892578, 3.15220308303833, 130.13182067871094, 128.699462890625, 41.92866516113281, 41.749027252197266, 29.97181510925293, 109.0545425415039, 43.700199127197266, 741.4284057617188, 368.6387939453125, 557.0759887695312]], "relative_norm": [[0.010859326459467411, 0.14137758314609528, 0.0027285348623991013, 0.11496899276971817, 0.12102581560611725, 0.03885428234934807, 0.040306273847818375, 0.030122356489300728, 0.11346358060836792, 0.03357396274805069, 0.7587447762489319, 0.3372013568878174, 0.5394566655158997]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 65431, 20539, 17692, 6633, 27696, 2274, 1852, 7748, 58829], "first_top_logits": [28.125, 27.625, 24.375, 21.75, 21.625, 21.375, 21.25, 20.625, 20.625, 20.5], "source_correct": false, "target_correct": false, "seconds": 0.48236870765686035, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.2182929515838623, "target_logp": -24.18396759033203, "log_odds": -20.96567463874817, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-1.0893245935440063, -2.127061367034912, -0.0019071976421400905]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-22.128387451171875, -2.0198562145233154, -0.03443560004234314, -0.0012898467248305678]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.70660400390625, 114.65323638916016, 5.455713748931885, 135.03790283203125, 128.2316131591797, 47.46877670288086, 39.820125579833984, 28.86901092529297, 106.69332885742188, 45.90440368652344, 743.6777954101562, 373.4056091308594, 559.7076416015625]], "relative_norm": [[0.01085657812654972, 0.1443376988172531, 0.004720335826277733, 0.11895038932561874, 0.12073754519224167, 0.04390691965818405, 0.038349393755197525, 0.029102379456162453, 0.11123519390821457, 0.035300713032484055, 0.7607352137565613, 0.3405301868915558, 0.5418066382408142]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7119750976562, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.70660400390625, 114.65323638916016, 5.455713748931885, 135.03790283203125, 128.2316131591797, 47.46877670288086, 39.820125579833984, 28.86901092529297, 106.69332885742188, 45.90440368652344, 743.6777954101562, 373.4056091308594, 559.7076416015625]], "relative_norm": [[0.01085657812654972, 0.1443376988172531, 0.004720335826277733, 0.11895038932561874, 0.12073754519224167, 0.04390691965818405, 0.038349393755197525, 0.029102379456162453, 0.11123519390821457, 0.035300713032484055, 0.7607352137565613, 0.3405301868915558, 0.5418066382408142]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysia Ringgit.  The currency used in the country containing", "output_ids": [27696, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 110.60438537597656, 3.079007148742676, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.760746002197266, 29.888771057128906, 109.08887481689453, 43.76106643676758, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.70660400390625, 110.59092712402344, 3.1979117393493652, 130.12908935546875, 128.78692626953125, 41.89407730102539, 41.72538757324219, 29.910255432128906, 109.12617492675781, 43.77950668334961, 741.47607421875, 368.640869140625, 556.9129028320312]], "relative_norm": [[0.01085657812654972, 0.141296848654747, 0.002768100006505847, 0.11496657878160477, 0.12110806256532669, 0.0388222299516201, 0.0402834489941597, 0.03006048873066902, 0.11353810876607895, 0.033634889870882034, 0.7587935328483582, 0.3372032642364502, 0.5392987132072449]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27696, 65431, 8394, 17692, 20539, 109123, 8853, 3639, 364, 22914], "first_top_logits": [23.625, 23.0, 19.875, 19.125, 18.5, 17.125, 16.875, 16.75, 16.625, 16.25], "source_correct": true, "target_correct": false, "seconds": 0.44142987485975027, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.240980863571167, "target_logp": -10.139143943786621, "log_odds": -7.898163080215454, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.1114915981888771, -2.1271212100982666, -0.002368028275668621]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.986491680145264, -2.129082202911377, -0.02254008688032627, -0.0010303908493369818]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73414611816406, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81276321411133, 28.874340057373047, 106.72412872314453, 45.86600112915039, 743.7119140625, 373.38031005859375, 559.681884765625]], "actual_norm": [[171.51402282714844, 114.59928131103516, 5.457086563110352, 134.96519470214844, 128.28225708007812, 47.50172805786133, 39.87526321411133, 28.86262321472168, 106.76673889160156, 45.970985412597656, 743.7692260742188, 373.3523864746094, 559.638916015625]], "relative_norm": [[0.010844402015209198, 0.14426977932453156, 0.004721523728221655, 0.11888633668422699, 0.12078522890806198, 0.04393739625811577, 0.0384024940431118, 0.02909594029188156, 0.11131172627210617, 0.035351917147636414, 0.7608287334442139, 0.34048163890838623, 0.5417401194572449]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73414611816406, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81276321411133, 28.874340057373047, 106.72412872314453, 45.86600112915039, 743.7119140625, 373.38031005859375, 559.681884765625]], "actual_norm": [[171.51402282714844, 114.59928131103516, 5.457086563110352, 134.96519470214844, 128.28225708007812, 47.50172805786133, 39.87526321411133, 28.86262321472168, 106.76673889160156, 45.970985412597656, 743.7692260742188, 373.3523864746094, 559.638916015625]], "relative_norm": [[0.010844402015209198, 0.14426977932453156, 0.004721523728221655, 0.11888633668422699, 0.12078522890806198, 0.04393739625811577, 0.0384024940431118, 0.02909594029188156, 0.11131172627210617, 0.035351917147636414, 0.7608287334442139, 0.34048163890838623, 0.5417401194572449]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  The currency used in the country containing", "output_ids": [65431, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73414611816406, 110.60437774658203, 3.0790069103240967, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.7607421875, 29.888769149780273, 109.0888671875, 43.76106262207031, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.51402282714844, 110.61360168457031, 3.2354867458343506, 130.17825317382812, 128.75611877441406, 41.948184967041016, 41.85132598876953, 29.87613296508789, 109.12995147705078, 43.8394660949707, 741.4623413085938, 368.66571044921875, 557.0479125976562]], "relative_norm": [[0.010844402015209198, 0.14132581651210785, 0.002800624817609787, 0.11501001566648483, 0.12107909470796585, 0.0388723723590374, 0.0404050350189209, 0.03002619370818138, 0.11354204267263412, 0.033680956810712814, 0.7587795257568359, 0.3372259736061096, 0.5394294857978821]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 7855, 20539, 27696, 2274, 21525, 26392, 109123, 431, 4882], "first_top_logits": [26.75, 23.875, 23.125, 21.75, 21.375, 21.25, 20.875, 20.875, 20.75, 20.625], "source_correct": true, "target_correct": false, "seconds": 0.4258963749743998, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.3964061737060547, "target_logp": -13.090339660644531, "log_odds": -10.693933486938477, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.26679402589797974, -2.127077102661133, -0.002534988336265087]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-11.454294204711914, -1.6032063961029053, -0.03151986747980118, -0.0013192531187087297]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.7341766357422, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81277084350586, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7120361328125, 373.38037109375, 559.681884765625]], "actual_norm": [[171.67010498046875, 114.62699127197266, 5.484793663024902, 134.9596710205078, 128.30166625976562, 47.42232131958008, 39.84345245361328, 28.918296813964844, 106.76324462890625, 45.87290954589844, 743.7368774414062, 373.4366149902344, 559.6976928710938]], "relative_norm": [[0.010854270309209824, 0.14430466294288635, 0.004745495971292257, 0.1188814714550972, 0.12080350518226624, 0.04386394843459129, 0.03837186098098755, 0.02915206365287304, 0.11130808293819427, 0.035276494920253754, 0.7607956528663635, 0.3405584692955017, 0.5417970418930054]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.7341766357422, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81277084350586, 28.87434196472168, 106.72413635253906, 45.866004943847656, 743.7120361328125, 373.38037109375, 559.681884765625]], "actual_norm": [[171.67010498046875, 114.62699127197266, 5.484793663024902, 134.9596710205078, 128.30166625976562, 47.42232131958008, 39.84345245361328, 28.918296813964844, 106.76324462890625, 45.87290954589844, 743.7368774414062, 373.4366149902344, 559.6976928710938]], "relative_norm": [[0.010854270309209824, 0.14430466294288635, 0.004745495971292257, 0.1188814714550972, 0.12080350518226624, 0.04386394843459129, 0.03837186098098755, 0.02915206365287304, 0.11130808293819427, 0.035276494920253754, 0.7607956528663635, 0.3405584692955017, 0.5417970418930054]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  The currency used in the country containing", "output_ids": [65431, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.7341766357422, 110.6043930053711, 3.079007148742676, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.760746002197266, 29.88877296447754, 109.08888244628906, 43.76106643676758, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.67010498046875, 110.63793182373047, 3.1610636711120605, 130.1280517578125, 128.7633056640625, 41.91059494018555, 41.77586364746094, 29.904508590698242, 109.09558868408203, 43.770843505859375, 741.4082641601562, 368.62530517578125, 556.9869384765625]], "relative_norm": [[0.010854270309209824, 0.14135690033435822, 0.0027362045366317034, 0.11496566236019135, 0.12108585238456726, 0.03883753716945648, 0.04033217951655388, 0.030054712668061256, 0.11350628733634949, 0.03362823650240898, 0.7587241530418396, 0.3371890187263489, 0.5393704175949097]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 1532, 20539, 17692, 21525, 2274, 3639, 10058, 27696, 7855], "first_top_logits": [25.125, 23.125, 22.25, 20.875, 20.625, 20.375, 20.375, 20.0, 19.875, 19.375], "source_correct": true, "target_correct": false, "seconds": 0.4653035830706358, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-29/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-29", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Penang", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Penang is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.2052488327026367, "target_logp": -11.669103622436523, "log_odds": -9.463854789733887, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.18522606790065765, -2.017836809158325, -0.002186171943321824]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-9.060226440429688, -2.581069231033325, -0.02668297290802002, -0.0011250602547079325]}}, "score_trace": {"source": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.874340057373047, 106.72413635253906, 45.866004943847656, 743.7119140625, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.54478454589844, 114.64838409423828, 5.466699123382568, 134.94229125976562, 128.26585388183594, 47.45680236816406, 39.84360122680664, 28.831005096435547, 106.76310729980469, 45.8729248046875, 743.5867309570312, 373.3609313964844, 559.7249755859375]], "relative_norm": [[0.010846346616744995, 0.14433158934116364, 0.004729840438812971, 0.11886616796255112, 0.12076978385448456, 0.043895844370126724, 0.03837200254201889, 0.029064064845442772, 0.11130794137716293, 0.03527650609612465, 0.7606420516967773, 0.3404894471168518, 0.5418234467506409]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.874340057373047, 106.72413635253906, 45.866004943847656, 743.7119140625, 373.3803405761719, 559.681884765625]], "actual_norm": [[171.54478454589844, 114.64838409423828, 5.466699123382568, 134.94229125976562, 128.26585388183594, 47.45680236816406, 39.84360122680664, 28.831005096435547, 106.76310729980469, 45.8729248046875, 743.5867309570312, 373.3609313964844, 559.7249755859375]], "relative_norm": [[0.010846346616744995, 0.14433158934116364, 0.004729840438812971, 0.11886616796255112, 0.12076978385448456, 0.043895844370126724, 0.03837200254201889, 0.029064064845442772, 0.11130794137716293, 0.03527650609612465, 0.7606420516967773, 0.3404894471168518, 0.5418234467506409]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  The currency used in the country containing", "output_ids": [65431, 21525, 12882, 13, 220, 576, 11413, 1483, 304, 279, 3146, 8482], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 13294, 524, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[171.73416137695312, 110.60437774658203, 3.0790069103240967, 130.14784240722656, 128.7285614013672, 41.91170120239258, 41.7607421875, 29.888769149780273, 109.08887481689453, 43.76106262207031, 741.3934326171875, 368.6042175292969, 556.9762573242188]], "actual_norm": [[171.54478454589844, 110.62703704833984, 3.1699445247650146, 130.11538696289062, 128.68394470214844, 41.91796112060547, 41.80495071411133, 29.90662384033203, 109.07971954345703, 43.773590087890625, 741.4134521484375, 368.642333984375, 556.8817749023438]], "relative_norm": [[0.010846346616744995, 0.14134298264980316, 0.0027438916731625795, 0.11495447158813477, 0.12101122736930847, 0.038844361901283264, 0.04036026448011398, 0.03005683794617653, 0.1134897768497467, 0.03363034501671791, 0.7587294578552246, 0.33720460534095764, 0.5392686128616333]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 7855, 21525, 17692, 27696, 10058, 4882, 13369, 3639], "first_top_logits": [26.5, 24.5, 22.0, 21.875, 21.625, 21.25, 20.375, 20.0, 19.875, 19.625], "source_correct": true, "target_correct": false, "seconds": 0.46728116599842906, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/readout", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 125.5, "target_rank": 166.5, "source_logit": 5.09375, "target_logit": 4.8125, "top_ids": [38352, 11413, 45418, 8330, 15973, 31491, 1111, 7065, 28453, 12430, 14042, 11766], "top_tokens": ["uenta", " currency", ".currency", " Australia", "currency", "uesto", "ule", "ican", " Currency", "aciones", "razil", " Standard"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 541.0, "target_rank": 2826.0, "source_logit": 3.359375, "target_logit": 1.265625, "top_ids": [11413, 96694, 15973, 3001, 18174, 53661, 3693, 28130, 21295, 4787, 7065, 28453], "top_tokens": [" currency", "onis", "currency", "ica", "oret", "cla", " American", "ija", "ollar", " North", "ican", " Currency"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 39.0, "target_rank": 30.0, "source_logit": 19.75, "target_logit": 20.125, "top_ids": [11413, 35602, 90475, 45418, 77186, 28453, 8330, 36911, 12463, 32488, 38664, 3263], "top_tokens": [" currency", " currencies", "(currency", ".currency", ".Currency", " Currency", " Australia", ".\".", " colour", " euros", " recognised", "\"."]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 112.5, "target_rank": 100.5, "source_logit": 14.5625, "target_logit": 14.6875, "top_ids": [11413, 7214, 37198, 65001, 74508, 35602, 13224, 16675, 47412, 8330, 7855, 21130], "top_tokens": [" currency", " foreign", " Commonwealth", "\"\u2014", "\u00bb.", " currencies", " Britain", " nations", " foreigners", " Australia", " British", " parliament"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 13227.5, "target_rank": 10120.5, "source_logit": -1.2578125, "target_logit": -0.7734375, "top_ids": [28453, 21251, 3731, 3776, 11413, 603, 55964, 4229, 5364, 5737, 11426, 45418], "top_tokens": [" Currency", "onia", " Red", " One", " currency", "one", "\u2011", " Ge", "ador", " Group", " Land", ".currency"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 8710.0, "target_rank": 7772.0, "source_logit": -1.2578125, "target_logit": -1.0703125, "top_ids": [364, 11766, 479, 2932, 362, 467, 7179, 11413, 356, 18174, 547, 198], "top_tokens": [" '", " Standard", " G", " She", " A", " W", " International", " currency", " C", "oret", " U", "\n"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 1249.5, "target_rank": 1787.5, "source_logit": 13.0, "target_logit": 12.4375, "top_ids": [11413, 35602, 13867, 28453, 3146, 5313, 7576, 30176, 3283, 60261, 29973, 32488], "top_tokens": [" currency", " currencies", " county", " Currency", " country", " national", " prices", " counties", " city", " tariff", " licence", " euros"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 178.5, "target_rank": 632.0, "source_logit": 14.0, "target_logit": 12.375, "top_ids": [11413, 37198, 105492, 94936, 5313, 35602, 7214, 13867, 11766, 3146, 6364, 5837], "top_tokens": [" currency", " Commonwealth", "\u4e2d\u534e\u4eba\u6c11", " denomination", " national", " currencies", " foreign", " county", " Standard", " country", " English", " countries"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 15432.0, "target_rank": 6366.5, "source_logit": -1.875, "target_logit": -0.23828125, "top_ids": [4787, 3693, 4229, 2240, 279, 6364, 5468, 46427, 5055, 89478, 5434, 7828], "top_tokens": [" North", " American", " Ge", " None", " the", " English", " ...\n\n", "/is", " National", "\u2026\n", "\u2026\n\n", " Bel"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 4562.0, "target_rank": 9405.0, "source_logit": 0.054931640625, "target_logit": -1.171875, "top_ids": [1852, 3693, 3639, 4102, 271, 479, 4787, 5055, 2701, 4882, 5938, 362], "top_tokens": [" same", " American", " United", "\u00a0", "\n\n", " G", " North", " National", " following", " South", " German", " A"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1654.5, "target_rank": 2646.5, "source_logit": 11.1875, "target_logit": 10.125, "top_ids": [11413, 35602, 104142, 28453, 103408, 113883, 26321, 32488, 90475, 101406, 106333, 77186], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u6b27\u5143", "\u672c\u56fd", "Currency", " euros", "(currency", "\u8d27\u5e01", "\u5916\u6c47", ".Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 970.5, "target_rank": 3949.5, "source_logit": 7.15625, "target_logit": 4.3125, "top_ids": [104142, 112397, 116866, 101237, 99599, 105492, 109442, 104773, 106333, 108594, 104531, 99563], "top_tokens": ["\u4eba\u6c11\u5e01", "\u56fd\u5bb6\u6807\u51c6", "\u76f4\u8f96\u5e02", "\u7f8e\u5143", "\u56fd\u5bb6", "\u4e2d\u534e\u4eba\u6c11", "\u8d22\u653f\u90e8", "\u5171\u548c\u56fd", "\u5916\u6c47", "\u53bf\u7ea7", "\u4eba\u6c11\u653f\u5e9c", "\u653f\u5e9c"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 35.0, "target_rank": 124.5, "source_logit": 15.0625, "target_logit": 10.5, "top_ids": [11413, 104142, 11192, 17692, 28453, 112455, 7855, 35602, 80189, 42364, 19551, 15973], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " dollar", " Currency", "\u82f1\u9551", " British", " currencies", " Dollars", " Dollar", " USD", "currency"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 9.0, "target_rank": 65.5, "source_logit": 18.5, "target_logit": 11.5625, "top_ids": [104142, 11413, 65431, 17692, 28453, 7855, 109123, 3639, 27696, 20539, 35602, 42364], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", " Malaysian", " dollar", " Currency", " British", "\u9a6c\u6765\u897f\u4e9a", " United", " Malaysia", " Singapore", " currencies", " Dollar"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 35.5, "target_rank": 116.0, "source_logit": 26.75, "target_logit": 19.75, "top_ids": [104142, 11413, 112455, 35602, 28453, 11192, 17692, 101406, 101237, 80189, 77186, 26321], "top_tokens": ["\u4eba\u6c11\u5e01", " currency", "\u82f1\u9551", " currencies", " Currency", " dollars", " dollar", "\u8d27\u5e01", "\u7f8e\u5143", " Dollars", ".Currency", "Currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 13.0, "target_rank": 59.5, "source_logit": 25.875, "target_logit": 17.375, "top_ids": [104142, 65431, 11413, 28453, 35602, 101406, 101237, 112455, 109123, 17692, 110065, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " Malaysian", " currency", " Currency", " currencies", "\u8d27\u5e01", "\u7f8e\u5143", "\u82f1\u9551", "\u9a6c\u6765\u897f\u4e9a", " dollar", "\u6c47\u7387", ".Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 10.0, "target_rank": 82.5, "source_logit": 20.5, "target_logit": 15.3125, "top_ids": [279, 7855, 20539, 65431, 3881, 10058, 3639, 21525, 109123, 27696, 5644, 2274], "top_tokens": [" the", " British", " Singapore", " Malaysian", " known", " ring", " United", " Ring", "\u9a6c\u6765\u897f\u4e9a", " Malaysia", " held", " US"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 5.0, "target_rank": 39.5, "source_logit": 22.5, "target_logit": 15.8125, "top_ids": [65431, 20539, 21525, 10058, 27696, 2274, 109123, 1532, 109122, 17692, 7855, 3639], "top_tokens": [" Malaysian", " Singapore", " Ring", " ring", " Malaysia", " US", "\u9a6c\u6765\u897f\u4e9a", " New", "\u9a6c\u6765", " dollar", " British", " United"]}], "source_token": 27696, "target_token": 23968, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Joh", "or", " Bah", "ru", " is", " the"], "seconds": 1.160706584341824}
{"run_id": "eval-30/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.38642954826355, "target_logp": -10.496665954589844, "log_odds": -8.110236406326294, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.14584864675998688, -2.237997055053711, -0.0025838580913841724]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.895848751068115, -2.5808138847351074, -0.01793278194963932, -0.002070189919322729]}}, "score_trace": {"source": [], "target": []}, "output": " Malaysian Ringgit.  What is the currency used in the", "output_ids": [65431, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [], "first_top_ids": [65431, 20539, 21525, 10058, 27696, 2274, 109123, 1532, 109122, 17692], "first_top_logits": [27.5, 25.0, 24.25, 22.875, 22.5, 21.25, 21.25, 20.75, 20.375, 20.375], "source_correct": true, "target_correct": false, "seconds": 0.524834165815264, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.38642954826355, "target_logp": -10.496665954589844, "log_odds": -8.110236406326294, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.14584864675998688, -2.237997055053711, -0.0025838580913841724]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.895848751068115, -2.5808138847351074, -0.01793278194963932, -0.002070189919322729]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  What is the currency used in the", "output_ids": [65431, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 21525, 10058, 27696, 2274, 109123, 1532, 109122, 17692], "first_top_logits": [27.5, 25.0, 24.25, 22.875, 22.5, 21.25, 21.25, 20.75, 20.375, 20.375], "source_correct": true, "target_correct": false, "seconds": 0.4358218749985099, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -31.116132736206055, "target_logp": -4.614652633666992, "log_odds": 26.501480102539062, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-29.09575843811035, -2.01785945892334, -0.0025164384860545397]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-2.1270077228546143, -2.465674638748169, -0.020371729508042336, -0.0015985103091225028]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6294860839844]], "actual_norm": [[171.71397399902344, 114.66758728027344, 5.452879905700684, 134.96002197265625, 128.23233032226562, 47.52134323120117, 39.822601318359375, 28.868017196655273, 106.67507934570312, 13.498698234558105, 605.331787109375, 195.56790161132812, 227.11007690429688, 370.6643981933594, 478.61041259765625]], "relative_norm": [[0.010857044719159603, 0.14435575902462006, 0.004717884119600058, 0.11888178437948227, 0.12073821574449539, 0.043955542147159576, 0.03835177794098854, 0.029101377353072166, 0.11121616512537003, 0.01191599853336811, 0.5852999687194824, 0.16095001995563507, 0.259067565202713, 0.33809250593185425, 0.4658338129520416]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6294860839844]], "actual_norm": [[171.71397399902344, 114.66758728027344, 5.452879905700684, 134.96002197265625, 128.23233032226562, 47.52134323120117, 39.822601318359375, 28.868017196655273, 106.67507934570312, 13.498698234558105, 605.331787109375, 195.56790161132812, 227.11007690429688, 370.6643981933594, 478.61041259765625]], "relative_norm": [[0.010857044719159603, 0.14435575902462006, 0.004717884119600058, 0.11888178437948227, 0.12073821574449539, 0.043955542147159576, 0.03835177794098854, 0.029101377353072166, 0.11121616512537003, 0.01191599853336811, 0.5852999687194824, 0.16095001995563507, 0.259067565202713, 0.33809250593185425, 0.4658338129520416]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesia Rupiah.\n\nInference: The currency used in", "output_ids": [23968, 431, 454, 19310, 382, 641, 2202, 25, 576, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 112.01326751708984, 6.302591800689697, 135.22486877441406, 127.96403503417969, 46.60271072387695, 41.33281326293945, 25.437015533447266, 105.1482925415039, 13.982972145080566, 607.4849243164062, 203.1835174560547, 226.53077697753906, 379.0821838378906, 482.4213562011719]], "actual_norm": [[171.71397399902344, 112.02200317382812, 6.34480094909668, 135.24932861328125, 127.94578552246094, 46.62255859375, 41.31288146972656, 25.478282928466797, 105.11576843261719, 14.114095687866211, 607.494384765625, 203.22862243652344, 226.52407836914062, 378.9949951171875, 482.5349426269531]], "relative_norm": [[0.010857044719159603, 0.14137952029705048, 0.005477261263877153, 0.11927888542413712, 0.12055493891239166, 0.04312743991613388, 0.039820995181798935, 0.025671811774373055, 0.10930608212947845, 0.01245795376598835, 0.5877973437309265, 0.16654711961746216, 0.2588837444782257, 0.34648314118385315, 0.47091880440711975]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23968, 58829, 2263, 111348, 84378, 17672, 19461, 20263, 1257, 7748], "first_top_logits": [36.25, 34.25, 26.125, 26.0, 22.875, 22.5, 22.25, 22.25, 22.0, 21.75], "source_correct": false, "target_correct": true, "seconds": 0.4608904169872403, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.297419786453247, "target_logp": -8.918726921081543, "log_odds": -5.621307134628296, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-1.1670076847076416, -2.1271581649780273, -0.0032540010288357735]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-6.542007923126221, -2.3520567417144775, -0.022834688425064087, -0.0018273575697094202]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.75006103515625, 114.65499114990234, 5.465886116027832, 134.89073181152344, 128.28697204589844, 47.506553649902344, 39.78873825073242, 28.91241455078125, 106.74829864501953, 13.453461647033691, 605.4056396484375, 195.5210418701172, 227.13607788085938, 370.61932373046875, 478.55780029296875]], "relative_norm": [[0.010859326459467411, 0.1443399041891098, 0.004729137290269136, 0.11882074922323227, 0.12078966945409775, 0.043941862881183624, 0.038319166749715805, 0.029146132990717888, 0.11129250377416611, 0.011876066215336323, 0.5853713750839233, 0.1609114557504654, 0.25909724831581116, 0.33805137872695923, 0.46578261256217957]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.75006103515625, 114.65499114990234, 5.465886116027832, 134.89073181152344, 128.28697204589844, 47.506553649902344, 39.78873825073242, 28.91241455078125, 106.74829864501953, 13.453461647033691, 605.4056396484375, 195.5210418701172, 227.13607788085938, 370.61932373046875, 478.55780029296875]], "relative_norm": [[0.010859326459467411, 0.1443399041891098, 0.004729137290269136, 0.11882074922323227, 0.12078966945409775, 0.043941862881183624, 0.038319166749715805, 0.029146132990717888, 0.11129250377416611, 0.011876066215336323, 0.5853713750839233, 0.1609114557504654, 0.25909724831581116, 0.33805137872695923, 0.46578261256217957]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Joh", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 26176], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 112.01326751708984, 6.302591323852539, 135.22486877441406, 127.96403503417969, 46.60271072387695, 41.33281707763672, 25.437015533447266, 105.1482925415039, 13.982973098754883, 607.4849243164062, 203.1835174560547, 226.53077697753906, 379.0821838378906, 482.42132568359375]], "actual_norm": [[171.75006103515625, 112.016845703125, 6.436271667480469, 135.16763305664062, 127.9871826171875, 46.62312698364258, 41.3362922668457, 25.47892189025879, 105.1555404663086, 14.019156455993652, 607.5057983398438, 203.1693115234375, 226.56944274902344, 379.0642395019531, 482.41522216796875]], "relative_norm": [[0.010859326459467411, 0.14137302339076996, 0.005556225311011076, 0.11920683830976486, 0.12059394270181656, 0.043127965182065964, 0.03984355926513672, 0.02567245624959469, 0.10934744030237198, 0.012374155223369598, 0.5878084301948547, 0.16649852693080902, 0.2589356005191803, 0.3465464413166046, 0.4708019495010376]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 65431, 20539, 6633, 58829, 10058, 27696, 2274, 21525, 17692], "first_top_logits": [28.0, 27.375, 24.375, 22.125, 21.875, 21.5, 21.375, 21.125, 21.0, 21.0], "source_correct": false, "target_correct": false, "seconds": 0.48293895879760385, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.1462910175323486, "target_logp": -23.008441925048828, "log_odds": -19.86215090751648, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.7928482890129089, -2.350405693054199, -0.003036889946088195]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-20.511598587036133, -2.4657208919525146, -0.02927587367594242, -0.0018461579456925392]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6294860839844]], "actual_norm": [[171.70660400390625, 114.65323638916016, 5.455713748931885, 135.03790283203125, 128.2316131591797, 47.46877670288086, 39.820125579833984, 28.86901092529297, 106.69332885742188, 13.502646446228027, 605.4378051757812, 195.56393432617188, 227.11245727539062, 370.6730651855469, 478.5935363769531]], "relative_norm": [[0.01085657812654972, 0.1443376988172531, 0.004720335826277733, 0.11895038932561874, 0.12073754519224167, 0.04390691965818405, 0.038349393755197525, 0.029102379456162453, 0.11123519390821457, 0.011919483542442322, 0.5854024887084961, 0.16094674170017242, 0.2590702772140503, 0.338100403547287, 0.465817391872406]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.87434196472168, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6294860839844]], "actual_norm": [[171.70660400390625, 114.65323638916016, 5.455713748931885, 135.03790283203125, 128.2316131591797, 47.46877670288086, 39.820125579833984, 28.86901092529297, 106.69332885742188, 13.502646446228027, 605.4378051757812, 195.56393432617188, 227.11245727539062, 370.6730651855469, 478.5935363769531]], "relative_norm": [[0.01085657812654972, 0.1443376988172531, 0.004720335826277733, 0.11895038932561874, 0.12073754519224167, 0.04390691965818405, 0.038349393755197525, 0.029102379456162453, 0.11123519390821457, 0.011919483542442322, 0.5854024887084961, 0.16094674170017242, 0.2590702772140503, 0.338100403547287, 0.465817391872406]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysia Ringgit.  What is the currency used in the", "output_ids": [27696, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 112.01326751708984, 6.302591800689697, 135.22486877441406, 127.96403503417969, 46.60271072387695, 41.33281326293945, 25.437015533447266, 105.1482925415039, 13.982972145080566, 607.4849243164062, 203.1835174560547, 226.53077697753906, 379.0821838378906, 482.4213562011719]], "actual_norm": [[171.70660400390625, 111.96365356445312, 6.351227283477783, 135.26873779296875, 127.95346069335938, 46.62997055053711, 41.33172607421875, 25.472064971923828, 105.11177062988281, 14.113646507263184, 607.41259765625, 203.18052673339844, 226.5062255859375, 379.06842041015625, 482.4783630371094]], "relative_norm": [[0.01085657812654972, 0.14130587875843048, 0.005482809152454138, 0.11929600685834885, 0.12056216597557068, 0.04313429817557335, 0.039839159697294235, 0.02566554583609104, 0.10930192470550537, 0.012457557953894138, 0.5877182483673096, 0.16650770604610443, 0.2588633596897125, 0.34655025601387024, 0.47086358070373535]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [27696, 65431, 8394, 20539, 17692, 109123, 3639, 18224, 364, 22914], "first_top_logits": [23.875, 23.75, 20.625, 18.875, 18.625, 17.75, 17.375, 17.125, 16.875, 16.75], "source_correct": true, "target_correct": false, "seconds": 0.47353029204532504, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.4096896648406982, "target_logp": -8.770681381225586, "log_odds": -6.360991716384888, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.16826845705509186, -2.2380614280700684, -0.003359865862876177]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-6.16826868057251, -2.5806777477264404, -0.020441459491848946, -0.001293775625526905]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73414611816406, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81276321411133, 28.874340057373047, 106.72412872314453, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.10891723632812, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.51402282714844, 114.59928131103516, 5.457086563110352, 134.96519470214844, 128.28225708007812, 47.50172805786133, 39.87526321411133, 28.86262321472168, 106.76673889160156, 13.459417343139648, 605.4417724609375, 195.5517120361328, 227.1508331298828, 370.66107177734375, 478.59356689453125]], "relative_norm": [[0.010844402015209198, 0.14426977932453156, 0.004721523728221655, 0.11888633668422699, 0.12078522890806198, 0.04393739625811577, 0.0384024940431118, 0.02909594029188156, 0.11131172627210617, 0.011881323531270027, 0.5854063034057617, 0.16093668341636658, 0.2591140568256378, 0.3380894660949707, 0.4658174216747284]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73414611816406, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81276321411133, 28.874340057373047, 106.72412872314453, 13.349225044250488, 605.4120483398438, 195.5902862548828, 227.10891723632812, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.51402282714844, 114.59928131103516, 5.457086563110352, 134.96519470214844, 128.28225708007812, 47.50172805786133, 39.87526321411133, 28.86262321472168, 106.76673889160156, 13.459417343139648, 605.4417724609375, 195.5517120361328, 227.1508331298828, 370.66107177734375, 478.59356689453125]], "relative_norm": [[0.010844402015209198, 0.14426977932453156, 0.004721523728221655, 0.11888633668422699, 0.12078522890806198, 0.04393739625811577, 0.0384024940431118, 0.02909594029188156, 0.11131172627210617, 0.011881323531270027, 0.5854063034057617, 0.16093668341636658, 0.2591140568256378, 0.3380894660949707, 0.4658174216747284]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  What is the currency used in the", "output_ids": [65431, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73414611816406, 112.01325988769531, 6.302591323852539, 135.22486877441406, 127.96402740478516, 46.60271072387695, 41.33281326293945, 25.437013626098633, 105.14828491210938, 13.982972145080566, 607.4849243164062, 203.1835174560547, 226.53077697753906, 379.0821533203125, 482.42132568359375]], "actual_norm": [[171.51402282714844, 112.00861358642578, 6.368498802185059, 135.28013610839844, 128.00552368164062, 46.626468658447266, 41.41508865356445, 25.45145034790039, 105.22319030761719, 14.136435508728027, 607.5494995117188, 203.21234130859375, 226.57546997070312, 379.12457275390625, 482.3463134765625]], "relative_norm": [[0.010844402015209198, 0.14136262238025665, 0.005497719161212444, 0.1193060576915741, 0.12061122804880142, 0.04313105717301369, 0.03991951048374176, 0.025644775480031967, 0.10941778123378754, 0.012477672658860683, 0.5878506898880005, 0.16653378307819366, 0.25894248485565186, 0.34660160541534424, 0.47073471546173096]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 7855, 21525, 109123, 27696, 431, 2274, 58829, 10058], "first_top_logits": [26.625, 23.875, 23.5, 22.625, 21.875, 21.875, 21.5, 21.125, 20.625, 20.625], "source_correct": true, "target_correct": false, "seconds": 0.43282141722738743, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.3900744915008545, "target_logp": -11.982386589050293, "log_odds": -9.592312097549438, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.25961172580718994, -2.1271371841430664, -0.0033257671166211367]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-9.822112083435059, -2.129286766052246, -0.028626425191760063, -0.0023613683879375458]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.7341766357422, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81277084350586, 28.87434196472168, 106.72413635253906, 13.349225997924805, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6295166015625]], "actual_norm": [[171.67010498046875, 114.62699127197266, 5.484793663024902, 134.9596710205078, 128.30166625976562, 47.42232131958008, 39.84345245361328, 28.918296813964844, 106.76324462890625, 13.468969345092773, 605.4016723632812, 195.5988006591797, 227.1293487548828, 370.7093200683594, 478.6579895019531]], "relative_norm": [[0.010854270309209824, 0.14430466294288635, 0.004745495971292257, 0.1188814714550972, 0.12080350518226624, 0.04386394843459129, 0.03837186098098755, 0.02915206365287304, 0.11130808293819427, 0.011889755725860596, 0.5853675603866577, 0.16097544133663177, 0.25908955931663513, 0.33813348412513733, 0.4658801257610321]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.7341766357422, 114.63069152832031, 5.382954120635986, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.81277084350586, 28.87434196472168, 106.72413635253906, 13.349225997924805, 605.4120483398438, 195.5902862548828, 227.1089324951172, 370.6179504394531, 478.6295166015625]], "actual_norm": [[171.67010498046875, 114.62699127197266, 5.484793663024902, 134.9596710205078, 128.30166625976562, 47.42232131958008, 39.84345245361328, 28.918296813964844, 106.76324462890625, 13.468969345092773, 605.4016723632812, 195.5988006591797, 227.1293487548828, 370.7093200683594, 478.6579895019531]], "relative_norm": [[0.010854270309209824, 0.14430466294288635, 0.004745495971292257, 0.1188814714550972, 0.12080350518226624, 0.04386394843459129, 0.03837186098098755, 0.02915206365287304, 0.11130808293819427, 0.011889755725860596, 0.5853675603866577, 0.16097544133663177, 0.25908955931663513, 0.33813348412513733, 0.4658801257610321]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  What is the currency used in the", "output_ids": [65431, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.7341766357422, 112.01326751708984, 6.3025922775268555, 135.22488403320312, 127.96403503417969, 46.60271072387695, 41.33281707763672, 25.437015533447266, 105.1482925415039, 13.982973098754883, 607.4849243164062, 203.18353271484375, 226.53077697753906, 379.0821838378906, 482.4213562011719]], "actual_norm": [[171.67010498046875, 112.04283142089844, 6.395403861999512, 135.26470947265625, 128.02330017089844, 46.597190856933594, 41.303977966308594, 25.512714385986328, 105.12340545654297, 14.044038772583008, 607.4766845703125, 203.18759155273438, 226.55230712890625, 379.0967712402344, 482.3895568847656]], "relative_norm": [[0.010854270309209824, 0.14140580594539642, 0.00552094541490078, 0.11929245293140411, 0.1206279769539833, 0.04310397431254387, 0.03981241211295128, 0.02570650540292263, 0.10931402444839478, 0.01239611767232418, 0.587780237197876, 0.16651350259780884, 0.2589160203933716, 0.34657618403434753, 0.47077691555023193]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 1532, 21525, 10058, 27696, 2274, 3639, 17692, 19551], "first_top_logits": [24.625, 22.375, 22.0, 21.75, 21.125, 20.0, 19.75, 19.75, 19.5, 18.375], "source_correct": true, "target_correct": false, "seconds": 0.4603406246751547, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-30/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-30", "split": "eval", "country": "Malaysia", "currency": "Malaysian ringgit", "aliases": ["ringgit", "MYR"], "city": "Johor Bahru", "target_country": "Indonesia", "target_currency": "Indonesian rupiah", "target_aliases": ["rupiah", "IDR"], "prompt": "Fact: The currency used in the country containing Johor Bahru is the", "source_continuation": " Malaysian ringgit", "target_continuation": " Indonesian rupiah"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.324984312057495, "target_logp": -10.771529197692871, "log_odds": -8.446544885635376, "token_scores": {"source": {"ids": [65431, 10058, 12882], "log_probs": [-0.19511303305625916, -2.1271681785583496, -0.0027032282669097185]}, "target": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.695113182067871, -3.050623893737793, -0.023977892473340034, -0.0018137922743335366]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.874340057373047, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.59027099609375, 227.10891723632812, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.54478454589844, 114.64838409423828, 5.466699123382568, 134.94229125976562, 128.26585388183594, 47.45680236816406, 39.84360122680664, 28.831005096435547, 106.76310729980469, 13.447914123535156, 605.4237670898438, 195.5933380126953, 227.1101531982422, 370.69696044921875, 478.621826171875]], "relative_norm": [[0.010846346616744995, 0.14433158934116364, 0.004729840438812971, 0.11886616796255112, 0.12076978385448456, 0.043895844370126724, 0.03837200254201889, 0.029064064845442772, 0.11130794137716293, 0.01187116838991642, 0.5853888988494873, 0.16097094118595123, 0.2590676546096802, 0.3381221890449524, 0.46584492921829224]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 19, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 114.63068389892578, 5.382953643798828, 134.9340362548828, 128.2685089111328, 47.467498779296875, 39.812767028808594, 28.874340057373047, 106.72413635253906, 13.349225044250488, 605.4120483398438, 195.59027099609375, 227.10891723632812, 370.6179504394531, 478.62945556640625]], "actual_norm": [[171.54478454589844, 114.64838409423828, 5.466699123382568, 134.94229125976562, 128.26585388183594, 47.45680236816406, 39.84360122680664, 28.831005096435547, 106.76310729980469, 13.447914123535156, 605.4237670898438, 195.5933380126953, 227.1101531982422, 370.69696044921875, 478.621826171875]], "relative_norm": [[0.010846346616744995, 0.14433158934116364, 0.004729840438812971, 0.11886616796255112, 0.12076978385448456, 0.043895844370126724, 0.03837200254201889, 0.029064064845442772, 0.11130794137716293, 0.01187116838991642, 0.5853888988494873, 0.16097094118595123, 0.2590676546096802, 0.3381221890449524, 0.46584492921829224]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Malaysian Ringgit.  What is the currency used in the", "output_ids": [65431, 21525, 12882, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 26176, 269, 31329, 2672, 374, 279], "generation_trace": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], "requested_norm": [[171.73416137695312, 112.01325988769531, 6.302591323852539, 135.22486877441406, 127.96402740478516, 46.60271072387695, 41.33281326293945, 25.437015533447266, 105.1482925415039, 13.982972145080566, 607.4849243164062, 203.1835174560547, 226.53076171875, 379.0821838378906, 482.42132568359375]], "actual_norm": [[171.54478454589844, 112.04322814941406, 6.3872222900390625, 135.21578979492188, 127.9614486694336, 46.54203796386719, 41.35068893432617, 25.470388412475586, 105.14529418945312, 14.055869102478027, 607.4649658203125, 203.16319274902344, 226.51271057128906, 379.1434326171875, 482.41864013671875]], "relative_norm": [[0.010846346616744995, 0.14140631258487701, 0.005513882264494896, 0.11924930661916733, 0.12056969851255417, 0.04305295646190643, 0.03985743597149849, 0.02566385641694069, 0.10933677852153778, 0.01240655966103077, 0.5877689123153687, 0.1664935052394867, 0.25887078046798706, 0.3466188609600067, 0.470805287361145]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [65431, 20539, 21525, 10058, 27696, 17692, 7855, 79140, 3639, 1532], "first_top_logits": [26.5, 24.5, 23.375, 21.875, 21.625, 21.25, 21.125, 20.125, 20.125, 19.875], "source_correct": true, "target_correct": false, "seconds": 0.45597504172474146, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/readout", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 31.5, "target_rank": 135.0, "source_logit": 6.40625, "target_logit": 4.96875, "top_ids": [38352, 15973, 11413, 8330, 45418, 31491, 7065, 56181, 3693, 6364, 28453, 7748], "top_tokens": ["uenta", "currency", " currency", " Australia", ".currency", "uesto", "ican", " Philippine", " American", " English", " Currency", " Indian"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 2199.0, "target_rank": 163.5, "source_logit": 1.7265625, "target_logit": 5.0, "top_ids": [11413, 15973, 96694, 3001, 28453, 21295, 53661, 3693, 45418, 11496, 5088, 7065], "top_tokens": [" currency", "currency", "onis", "ica", " Currency", "ollar", "cla", " American", ".currency", "ika", "urrency", "ican"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 6.0, "target_rank": 18.5, "source_logit": 23.375, "target_logit": 21.75, "top_ids": [11413, 35602, 90475, 45418, 77186, 23968, 58829, 104142, 28453, 7214, 8330, 32488], "top_tokens": [" currency", " currencies", "(currency", ".currency", ".Currency", " Indonesia", " Indonesian", "\u4eba\u6c11\u5e01", " Currency", " foreign", " Australia", " euros"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 55.5, "target_rank": 12.0, "source_logit": 16.125, "target_logit": 17.75, "top_ids": [11413, 7214, 35602, 3220, 47412, 15973, 32284, 78253, 17692, 65001, 5313, 74508], "top_tokens": [" currency", " foreign", " currencies", " money", " foreigners", "currency", " monetary", " treasury", " dollar", "\"\u2014", " national", "\u00bb."]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 5188.5, "target_rank": 2542.5, "source_logit": 0.33984375, "target_logit": 1.328125, "top_ids": [21251, 3776, 3731, 603, 28453, 7748, 55964, 12751, 56537, 9127, 11413, 19833], "top_tokens": ["onia", " One", " Red", "one", " Currency", " Indian", "\u2011", " Roman", " _______,", " Rock", " currency", " Amb"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 4417.5, "target_rank": 193.0, "source_logit": -0.057861328125, "target_logit": 4.3125, "top_ids": [7179, 479, 11413, 364, 18174, 11766, 64, 362, 356, 547, 198, 4102], "top_tokens": [" International", " G", " currency", " '", "oret", " Standard", "a", " A", " C", " U", "\n", "\u00a0"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 887.5, "target_rank": 163.0, "source_logit": 13.75, "target_logit": 16.0, "top_ids": [11413, 35602, 13867, 28453, 3146, 5313, 7576, 3283, 5837, 11192, 7214, 30176], "top_tokens": [" currency", " currencies", " county", " Currency", " country", " national", " prices", " city", " countries", " dollars", " foreign", " counties"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 322.5, "target_rank": 10.0, "source_logit": 13.5625, "target_logit": 17.125, "top_ids": [11413, 35602, 94936, 7214, 105492, 6489, 37198, 5313, 5837, 6323, 3146, 5297], "top_tokens": [" currency", " currencies", " denomination", " foreign", "\u4e2d\u534e\u4eba\u6c11", " international", " Commonwealth", " national", " countries", " Japan", " country", " standard"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 3981.5, "target_rank": 337.0, "source_logit": 0.470703125, "target_logit": 3.953125, "top_ids": [3693, 279, 89478, 2598, 4787, 4229, 5434, 1083, 46427, 4102, 15503, 271], "top_tokens": [" American", " the", "\u2026\n", " called", " North", " Ge", "\u2026\n\n", " also", "/is", "\u00a0", " primarily", "\n\n"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 4801.0, "target_rank": 270.5, "source_logit": -0.064453125, "target_logit": 4.09375, "top_ids": [1852, 271, 4102, 3693, 5938, 4882, 7179, 3639, 4787, 479, 362, 5055], "top_tokens": [" same", "\n\n", "\u00a0", " American", " German", " South", " International", " United", " North", " G", " A", " National"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1436.0, "target_rank": 730.5, "source_logit": 12.0625, "target_logit": 13.5, "top_ids": [11413, 35602, 104142, 28453, 103408, 113883, 90475, 26321, 101237, 77186, 106333, 101406], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u6b27\u5143", "\u672c\u56fd", "(currency", "Currency", "\u7f8e\u5143", ".Currency", "\u5916\u6c47", "\u8d27\u5e01"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 619.0, "target_rank": 422.0, "source_logit": 9.0, "target_logit": 9.75, "top_ids": [104142, 116866, 101237, 112397, 106333, 110065, 113883, 103408, 105492, 99599, 101406, 109442], "top_tokens": ["\u4eba\u6c11\u5e01", "\u76f4\u8f96\u5e02", "\u7f8e\u5143", "\u56fd\u5bb6\u6807\u51c6", "\u5916\u6c47", "\u6c47\u7387", "\u672c\u56fd", "\u6b27\u5143", "\u4e2d\u534e\u4eba\u6c11", "\u56fd\u5bb6", "\u8d27\u5e01", "\u8d22\u653f\u90e8"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 5.0, "target_rank": 221.0, "source_logit": 21.25, "target_logit": 8.4375, "top_ids": [11413, 58829, 104142, 11192, 23968, 17692, 28453, 80189, 2598, 35602, 15973, 19551], "top_tokens": [" currency", " Indonesian", "\u4eba\u6c11\u5e01", " dollars", " Indonesia", " dollar", " Currency", " Dollars", " called", " currencies", "currency", " USD"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 245.5, "source_logit": 25.25, "target_logit": 7.53125, "top_ids": [58829, 23968, 11413, 104142, 17692, 111348, 25662, 28453, 63033, 23234, 5313, 65431], "top_tokens": [" Indonesian", " Indonesia", " currency", "\u4eba\u6c11\u5e01", " dollar", "\u5370\u5c3c", " Netherlands", " Currency", " Jakarta", " Dutch", " national", " Malaysian"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 9.0, "target_rank": 261.5, "source_logit": 33.75, "target_logit": 16.5, "top_ids": [11413, 104142, 35602, 28453, 11192, 17692, 58829, 80189, 23968, 101406, 15973, 34710], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " Currency", " dollars", " dollar", " Indonesian", " Dollars", " Indonesia", "\u8d27\u5e01", "currency", "_currency"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 3.0, "target_rank": 325.5, "source_logit": 33.25, "target_logit": 11.75, "top_ids": [58829, 104142, 23968, 11413, 111348, 35602, 28453, 20263, 101406, 45418, 17692, 110065], "top_tokens": [" Indonesian", "\u4eba\u6c11\u5e01", " Indonesia", " currency", "\u5370\u5c3c", " currencies", " Currency", " Indones", "\u8d27\u5e01", ".currency", " dollar", "\u6c47\u7387"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 5.5, "target_rank": 1161.5, "source_logit": 23.0, "target_logit": 10.0, "top_ids": [279, 58829, 111348, 2598, 23968, 3881, 431, 2263, 7748, 220, 435, 19461], "top_tokens": [" the", " Indonesian", "\u5370\u5c3c", " called", " Indonesia", " known", " R", " Ind", " Indian", " ", " r", " Euro"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 1319.5, "source_logit": 25.125, "target_logit": 8.5, "top_ids": [58829, 23968, 431, 7748, 435, 2263, 111348, 11413, 19461, 11122, 5429, 17672], "top_tokens": [" Indonesian", " Indonesia", " R", " Indian", " r", " Ind", "\u5370\u5c3c", " currency", " Euro", " ru", " Republic", " euro"]}], "source_token": 23968, "target_token": 6323, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Sur", "ab", "aya", " is", " the"], "seconds": 1.1522054579108953}
{"run_id": "eval-31/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1928086280822754, "target_logp": -13.63892936706543, "log_odds": -11.446120738983154, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.03740682080388069, -2.129934310913086, -0.024550342932343483, -0.0009168950491584837]}, "target": {"ids": [10769, 57340], "log_probs": [-12.037406921386719, -1.6015220880508423]}}, "score_trace": {"source": [], "target": []}, "output": " Indonesian Rupiah.\n\nFact: The currency used in the", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [], "first_top_ids": [58829, 23968, 431, 7748, 435, 2263, 111348, 11413, 19461, 11122], "first_top_logits": [29.25, 25.125, 24.625, 23.875, 23.625, 23.0, 22.625, 21.125, 20.75, 20.5], "source_correct": true, "target_correct": false, "seconds": 0.4889815840870142, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1928086280822754, "target_logp": -13.63892936706543, "log_odds": -11.446120738983154, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.03740682080388069, -2.129934310913086, -0.024550342932343483, -0.0009168950491584837]}, "target": {"ids": [10769, 57340], "log_probs": [-12.037406921386719, -1.6015220880508423]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The currency used in the", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 23968, 431, 7748, 435, 2263, 111348, 11413, 19461, 11122], "first_top_logits": [29.25, 25.125, 24.625, 23.875, 23.625, 23.0, 22.625, 21.125, 20.75, 20.5], "source_correct": true, "target_correct": false, "seconds": 0.4848583750426769, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -40.01987838745117, "target_logp": -3.931264877319336, "log_odds": 36.088613510131836, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-37.957611083984375, -2.0202276706695557, -0.040894508361816406, -0.0011443501571193337]}, "target": {"ids": [10769, 57340], "log_probs": [-2.0201103687286377, -1.9111543893814087]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.7390899658203, 242.70455932617188, 180.9784393310547, 99.592529296875, 102.36357879638672, 101.82274627685547, 97.99144744873047, 196.9324951171875, 77.5084457397461, 90.45890045166016, 1016.7472534179688, 439.63104248046875, 944.0277709960938, 1181.1964111328125]], "relative_norm": [[0.015094874426722527, 0.30554232001304626, 0.15658427774906158, 0.08772774040699005, 0.096381276845932, 0.09418239444494247, 0.09437219798564911, 0.19852443039417267, 0.08080793172121048, 0.08564306050539017, 0.9142587780952454, 0.4811093211174011, 0.8570553660392761, 1.1293216943740845]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.7390899658203, 242.70455932617188, 180.9784393310547, 99.592529296875, 102.36357879638672, 101.82274627685547, 97.99144744873047, 196.9324951171875, 77.5084457397461, 90.45890045166016, 1016.7472534179688, 439.63104248046875, 944.0277709960938, 1181.1964111328125]], "relative_norm": [[0.015094874426722527, 0.30554232001304626, 0.15658427774906158, 0.08772774040699005, 0.096381276845932, 0.09418239444494247, 0.09437219798564911, 0.19852443039417267, 0.08080793172121048, 0.08564306050539017, 0.9142587780952454, 0.4811093211174011, 0.8570553660392761, 1.1293216943740845]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japan Shinkin.  What is the currency used in", "output_ids": [6323, 1417, 766, 258, 13, 220, 3555, 374, 279, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88668823242188, 78.78361511230469, 93.67924499511719, 1013.0532836914062, 428.2940368652344, 926.8455810546875, 1155.7559814453125]], "actual_norm": [[238.7390899658203, 238.94064331054688, 178.7091827392578, 99.06291198730469, 105.41584014892578, 101.10234069824219, 99.30131530761719, 202.90626525878906, 78.79095458984375, 93.73214721679688, 1013.2694091796875, 428.2995300292969, 926.87841796875, 1155.6845703125]], "relative_norm": [[0.015094874426722527, 0.30528324842453003, 0.1546899825334549, 0.08752020448446274, 0.09913046658039093, 0.09368909895420074, 0.09586968272924423, 0.20392541587352753, 0.08197645097970963, 0.0886877104640007, 0.9111427068710327, 0.4686187207698822, 0.8407103419303894, 1.1061830520629883]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [6323, 10769, 809, 57340, 619, 3639, 2274, 7855, 33554, 17692], "first_top_logits": [31.75, 30.25, 26.125, 24.0, 23.5, 23.375, 22.25, 21.375, 21.375, 21.375], "source_correct": false, "target_correct": false, "seconds": 0.49858195800334215, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -8.962312698364258, "target_logp": -18.830961227416992, "log_odds": -9.868648529052734, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-7.128162384033203, -1.8080075979232788, -0.024997718632221222, -0.0011451836908236146]}, "target": {"ids": [10769, 57340], "log_probs": [-17.128162384033203, -1.7027980089187622]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98483276367188, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51144409179688, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.64486694335938, 242.7481689453125, 180.9066925048828, 99.57714080810547, 102.39144134521484, 101.81426239013672, 97.92858123779297, 196.97702026367188, 77.55121612548828, 90.42699432373047, 1016.7583618164062, 439.6603698730469, 943.9246826171875, 1181.1265869140625]], "relative_norm": [[0.015088916756212711, 0.3055972456932068, 0.15652219951152802, 0.08771418035030365, 0.09640751779079437, 0.09417454898357391, 0.0943116545677185, 0.19856931269168854, 0.08085251599550247, 0.08561284840106964, 0.9142687320709229, 0.4811413884162903, 0.8569617867469788, 1.129254937171936]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98483276367188, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51144409179688, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.64486694335938, 242.7481689453125, 180.9066925048828, 99.57714080810547, 102.39144134521484, 101.81426239013672, 97.92858123779297, 196.97702026367188, 77.55121612548828, 90.42699432373047, 1016.7583618164062, 439.6603698730469, 943.9246826171875, 1181.1265869140625]], "relative_norm": [[0.015088916756212711, 0.3055972456932068, 0.15652219951152802, 0.08771418035030365, 0.09640751779079437, 0.09417454898357391, 0.0943116545677185, 0.19856931269168854, 0.08085251599550247, 0.08561284840106964, 0.9142687320709229, 0.4811413884162903, 0.8569617867469788, 1.129254937171936]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Sur", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 8082], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88670349121094, 78.78361511230469, 93.67925262451172, 1013.0533447265625, 428.29400634765625, 926.8455200195312, 1155.7559814453125]], "actual_norm": [[238.64486694335938, 238.9307861328125, 178.7126922607422, 99.0480728149414, 105.37840270996094, 101.12916564941406, 99.30152893066406, 202.8852996826172, 78.76947784423828, 93.74147033691406, 1013.113525390625, 428.2882995605469, 927.0078735351562, 1155.78076171875]], "relative_norm": [[0.015088916756212711, 0.3052706718444824, 0.15469300746917725, 0.08750709146261215, 0.09909526258707047, 0.09371395409107208, 0.09586988389492035, 0.20390434563159943, 0.08195410668849945, 0.08869653195142746, 0.911002516746521, 0.46860644221305847, 0.8408277630805969, 1.1062750816345215]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 58829, 6633, 220, 89782, 2263, 23968, 1257, 134597, 198], "first_top_logits": [27.875, 21.125, 21.0, 19.0, 19.0, 18.5, 18.375, 18.25, 18.125, 17.875], "source_correct": false, "target_correct": false, "seconds": 0.41633779229596257, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.75055193901062, "target_logp": -46.53154754638672, "log_odds": -42.7809956073761, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-1.600568175315857, -2.1297929286956787, -0.019168144091963768, -0.001022769371047616]}, "target": {"ids": [10769, 57340], "log_probs": [-44.72556686401367, -1.8059825897216797]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.76498413085938, 242.70928955078125, 180.94493103027344, 99.6073226928711, 102.33980560302734, 101.7933349609375, 97.9751205444336, 196.9191436767578, 77.49484252929688, 90.43394470214844, 1016.619140625, 439.6743469238281, 944.0322875976562, 1181.1689453125]], "relative_norm": [[0.0150965116918087, 0.3055482804775238, 0.15655529499053955, 0.08774077147245407, 0.09635889530181885, 0.0941551923751831, 0.094356469810009, 0.19851097464561462, 0.08079374581575394, 0.08561942726373672, 0.9141435623168945, 0.48115670680999756, 0.8570594787597656, 1.1292953491210938]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684936523438, 1016.7648315429688, 439.6617126464844, 943.9922485351562, 1181.1217041015625]], "actual_norm": [[238.76498413085938, 242.70928955078125, 180.94493103027344, 99.6073226928711, 102.33980560302734, 101.7933349609375, 97.9751205444336, 196.9191436767578, 77.49484252929688, 90.43394470214844, 1016.619140625, 439.6743469238281, 944.0322875976562, 1181.1689453125]], "relative_norm": [[0.0150965116918087, 0.3055482804775238, 0.15655529499053955, 0.08774077147245407, 0.09635889530181885, 0.0941551923751831, 0.094356469810009, 0.19851097464561462, 0.08079374581575394, 0.08561942726373672, 0.9141435623168945, 0.48115670680999756, 0.8570594787597656, 1.1292953491210938]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesia Rupiah.\n\nFact: The currency used in the", "output_ids": [23968, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88668823242188, 78.78361511230469, 93.67924499511719, 1013.0532836914062, 428.2940368652344, 926.8455810546875, 1155.7559814453125]], "actual_norm": [[238.76498413085938, 238.98239135742188, 178.67642211914062, 99.09142303466797, 105.44664001464844, 101.12164306640625, 99.24496459960938, 202.9426727294922, 78.762939453125, 93.7029037475586, 1013.079833984375, 428.2913513183594, 926.8936157226562, 1155.6630859375]], "relative_norm": [[0.0150965116918087, 0.305336594581604, 0.154661625623703, 0.08754539489746094, 0.09915943443775177, 0.09370698779821396, 0.0958152785897255, 0.20396201312541962, 0.08194729685783386, 0.08866003900766373, 0.9109722375869751, 0.4686097800731659, 0.8407241106033325, 1.106162428855896]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23968, 58829, 84378, 111348, 20263, 1257, 11413, 330, 364, 1425], "first_top_logits": [27.5, 26.5, 18.75, 16.375, 15.875, 15.6875, 15.1875, 14.3125, 14.125, 13.875], "source_correct": true, "target_correct": false, "seconds": 0.4502954580821097, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.475208282470703, "target_logp": -12.780364990234375, "log_odds": -10.305156707763672, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.4316798746585846, -2.020566701889038, -0.022011850029230118, -0.000949885172303766]}, "target": {"ids": [10769, 57340], "log_probs": [-10.869179725646973, -1.9111852645874023]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98483276367188, 99.60225677490234, 102.37149047851562, 101.80586242675781, 97.97480773925781, 196.9394989013672, 77.51143646240234, 90.42684173583984, 1016.7647705078125, 439.66168212890625, 943.9921875, 1181.1217041015625]], "actual_norm": [[238.40589904785156, 242.71466064453125, 180.92703247070312, 99.69169616699219, 102.40999603271484, 101.88557434082031, 98.0350341796875, 196.90589904785156, 77.48518371582031, 90.42320251464844, 1016.8010864257812, 439.721923828125, 943.9701538085938, 1181.123779296875]], "relative_norm": [[0.015073807910084724, 0.3055550456047058, 0.15653979778289795, 0.08781509101390839, 0.09642498195171356, 0.09424050897359848, 0.09441417455673218, 0.19849762320518494, 0.0807836726307869, 0.08560925722122192, 0.914307177066803, 0.48120877146720886, 0.8570030927658081, 1.1292521953582764]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98483276367188, 99.60225677490234, 102.37149047851562, 101.80586242675781, 97.97480773925781, 196.9394989013672, 77.51143646240234, 90.42684173583984, 1016.7647705078125, 439.66168212890625, 943.9921875, 1181.1217041015625]], "actual_norm": [[238.40589904785156, 242.71466064453125, 180.92703247070312, 99.69169616699219, 102.40999603271484, 101.88557434082031, 98.0350341796875, 196.90589904785156, 77.48518371582031, 90.42320251464844, 1016.8010864257812, 439.721923828125, 943.9701538085938, 1181.123779296875]], "relative_norm": [[0.015073807910084724, 0.3055550456047058, 0.15653979778289795, 0.08781509101390839, 0.09642498195171356, 0.09424050897359848, 0.09441417455673218, 0.19849762320518494, 0.0807836726307869, 0.08560925722122192, 0.914307177066803, 0.48120877146720886, 0.8570030927658081, 1.1292521953582764]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.  What is the currency used in", "output_ids": [58829, 431, 454, 19310, 13, 220, 3555, 374, 279, 11413, 1483, 304], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 238.9718017578125, 178.68426513671875, 99.06647491455078, 105.40560913085938, 101.10248565673828, 99.27228546142578, 202.8866729736328, 78.78361511230469, 93.67923736572266, 1013.05322265625, 428.29400634765625, 926.8455810546875, 1155.7559814453125]], "actual_norm": [[238.40589904785156, 238.9921112060547, 178.6936798095703, 99.10761260986328, 105.48545837402344, 101.21173095703125, 99.34675598144531, 202.90170288085938, 78.71041870117188, 93.73782348632812, 1013.1160888671875, 428.3307800292969, 926.7969970703125, 1155.8885498046875]], "relative_norm": [[0.015073807910084724, 0.3053490221500397, 0.15467655658721924, 0.08755969256162643, 0.0991959348320961, 0.09379047155380249, 0.09591355174779892, 0.20392084121704102, 0.08189265429973602, 0.08869308233261108, 0.9110048413276672, 0.4686529338359833, 0.8406364917755127, 1.1063783168792725]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 19461, 17672, 431, 7748, 23234, 11413, 2263, 14533, 17692], "first_top_logits": [22.75, 21.375, 20.625, 19.25, 19.125, 18.875, 18.75, 18.5, 17.375, 17.375], "source_correct": true, "target_correct": false, "seconds": 0.4659440000541508, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.0384559631347656, "target_logp": -9.875435829162598, "log_odds": -6.836979866027832, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-1.4012309312820435, -1.6035202741622925, -0.03251587972044945, -0.00118900160305202]}, "target": {"ids": [10769, 57340], "log_probs": [-7.963730812072754, -1.9117048978805542]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66781616210938, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587768554688, 97.97481536865234, 196.93951416015625, 77.51144409179688, 90.42684936523438, 1016.764892578125, 439.6617431640625, 943.9922485351562, 1181.121826171875]], "actual_norm": [[238.5471649169922, 242.68341064453125, 180.9674530029297, 99.68280792236328, 102.44883728027344, 101.87799835205078, 97.96182250976562, 196.9371337890625, 77.51385498046875, 90.40711212158203, 1016.8338012695312, 439.64434814453125, 944.0569458007812, 1180.988525390625]], "relative_norm": [[0.015082739293575287, 0.30551570653915405, 0.1565747708082199, 0.08780726045370102, 0.09646155685186386, 0.09423349797725677, 0.09434366226196289, 0.19852910935878754, 0.08081356436014175, 0.08559402823448181, 0.9143365621566772, 0.4811238646507263, 0.8570818901062012, 1.1291228532791138]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66781616210938, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587768554688, 97.97481536865234, 196.93951416015625, 77.51144409179688, 90.42684936523438, 1016.764892578125, 439.6617431640625, 943.9922485351562, 1181.121826171875]], "actual_norm": [[238.5471649169922, 242.68341064453125, 180.9674530029297, 99.68280792236328, 102.44883728027344, 101.87799835205078, 97.96182250976562, 196.9371337890625, 77.51385498046875, 90.40711212158203, 1016.8338012695312, 439.64434814453125, 944.0569458007812, 1180.988525390625]], "relative_norm": [[0.015082739293575287, 0.30551570653915405, 0.1565747708082199, 0.08780726045370102, 0.09646155685186386, 0.09423349797725677, 0.09434366226196289, 0.19852910935878754, 0.08081356436014175, 0.08559402823448181, 0.9143365621566772, 0.4811238646507263, 0.8570818901062012, 1.1291228532791138]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The currency used in the", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66781616210938, 238.97181701660156, 178.6842803955078, 99.06649017333984, 105.40560913085938, 101.10249328613281, 99.27229309082031, 202.88670349121094, 78.78361511230469, 93.67925262451172, 1013.0533447265625, 428.2940673828125, 926.8456420898438, 1155.7559814453125]], "actual_norm": [[238.5471649169922, 238.9964141845703, 178.61512756347656, 99.02792358398438, 105.49221801757812, 101.14620971679688, 99.22235870361328, 202.85032653808594, 78.80096435546875, 93.7193603515625, 1013.0487060546875, 428.25079345703125, 926.8067016601562, 1155.7630615234375]], "relative_norm": [[0.015082739293575287, 0.305354505777359, 0.1546085625886917, 0.0874892920255661, 0.09920229017734528, 0.09372974932193756, 0.09579344838857651, 0.20386919379234314, 0.08198685944080353, 0.0886756107211113, 0.9109442234039307, 0.4685654044151306, 0.8406452536582947, 1.1062581539154053]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 13, 19461, 892, 3639, 17672, 23968, 435, 304, 2274], "first_top_logits": [18.875, 18.5, 18.0, 17.375, 17.0, 16.75, 16.375, 16.25, 16.125, 16.125], "source_correct": true, "target_correct": false, "seconds": 0.48268712498247623, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-31/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-31", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Surabaya", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Surabaya is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.641721725463867, "target_logp": -15.01604175567627, "log_odds": -12.374320030212402, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.03967391327023506, -2.581083297729492, -0.020082945004105568, -0.0008817598572932184]}, "target": {"ids": [10769, 57340], "log_probs": [-12.664673805236816, -2.3513681888580322]}}, "score_trace": {"source": [{"sequence_length": 18, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684173583984, 1016.7647705078125, 439.66168212890625, 943.9921875, 1181.1217041015625]], "actual_norm": [[238.41529846191406, 242.6988067626953, 180.9642791748047, 99.620361328125, 102.40231323242188, 101.79595184326172, 97.93122100830078, 196.9240264892578, 77.58275604248047, 90.42369079589844, 1016.6361083984375, 439.7002258300781, 944.048583984375, 1181.017333984375]], "relative_norm": [[0.015074402093887329, 0.30553507804870605, 0.15657202899456024, 0.08775225281715393, 0.09641774743795395, 0.0941576138138771, 0.09431419521570206, 0.1985158920288086, 0.0808854028582573, 0.08560971915721893, 0.914158821105957, 0.48118501901626587, 0.8570742607116699, 1.129150390625]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 16, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 90.42684173583984, 1016.7647705078125, 439.66168212890625, 943.9921875, 1181.1217041015625]], "actual_norm": [[238.41529846191406, 242.6988067626953, 180.9642791748047, 99.620361328125, 102.40231323242188, 101.79595184326172, 97.93122100830078, 196.9240264892578, 77.58275604248047, 90.42369079589844, 1016.6361083984375, 439.7002258300781, 944.048583984375, 1181.017333984375]], "relative_norm": [[0.015074402093887329, 0.30553507804870605, 0.15657202899456024, 0.08775225281715393, 0.09641774743795395, 0.0941576138138771, 0.09431419521570206, 0.1985158920288086, 0.0808854028582573, 0.08560971915721893, 0.914158821105957, 0.48118501901626587, 0.8570742607116699, 1.129150390625]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The currency used in the", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 8082, 370, 12589, 374, 279], "generation_trace": [{"sequence_length": 14, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "requested_norm": [[238.66778564453125, 238.9718017578125, 178.68426513671875, 99.06647491455078, 105.40560913085938, 101.10247802734375, 99.27229309082031, 202.88668823242188, 78.78361511230469, 93.67924499511719, 1013.05322265625, 428.2940368652344, 926.8455200195312, 1155.7559814453125]], "actual_norm": [[238.41529846191406, 238.98301696777344, 178.67726135253906, 99.08872985839844, 105.41427612304688, 101.12664794921875, 99.25846099853516, 202.906494140625, 78.79971313476562, 93.65365600585938, 1013.1292724609375, 428.2459716796875, 926.9237060546875, 1155.683837890625]], "relative_norm": [[0.015074402093887329, 0.30533739924430847, 0.1546623408794403, 0.08754301071166992, 0.09912899881601334, 0.09371162205934525, 0.09582830965518951, 0.20392565429210663, 0.08198556303977966, 0.08861344307661057, 0.9110166430473328, 0.468560129404068, 0.8407514095306396, 1.106182336807251]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 23968, 4882, 4492, 19551, 45862, 2263, 111348, 13387, 26333], "first_top_logits": [25.5, 22.0, 18.5, 18.375, 18.375, 18.125, 17.75, 17.5, 17.375, 17.125], "source_correct": true, "target_correct": false, "seconds": 0.4499983750283718, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/readout", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "readouts": [{"layer": 9, "position": -2, "method": "logit", "source_rank": 20.5, "target_rank": 77.5, "source_logit": 7.28125, "target_logit": 5.78125, "top_ids": [15973, 11413, 8330, 38352, 45418, 6364, 56181, 7065, 31491, 3693, 14042, 71009], "top_tokens": ["currency", " currency", " Australia", "uenta", ".currency", " English", " Philippine", "ican", "uesto", " American", "razil", "/english"]}, {"layer": 9, "position": -1, "method": "logit", "source_rank": 1047.5, "target_rank": 78.0, "source_logit": 3.09375, "target_logit": 6.65625, "top_ids": [11413, 15973, 28453, 5088, 26321, 21295, 7065, 45418, 96694, 3693, 42189, 53661], "top_tokens": [" currency", "currency", " Currency", "urrency", "Currency", "ollar", "ican", ".currency", "onis", " American", "etz", "cla"]}, {"layer": 9, "position": -2, "method": "j", "source_rank": 4.0, "target_rank": 13.0, "source_logit": 25.125, "target_logit": 23.625, "top_ids": [11413, 35602, 3263, 23968, 22956, 58829, 24357, 7214, 8330, 1189, 12922, 34274], "top_tokens": [" currency", " currencies", "\".", " Indonesia", "\".\n", " Indonesian", " overseas", " foreign", " Australia", ".\"", " island", " Caribbean"]}, {"layer": 9, "position": -1, "method": "j", "source_rank": 83.0, "target_rank": 17.0, "source_logit": 16.375, "target_logit": 18.5, "top_ids": [11413, 35602, 7214, 3220, 34710, 17692, 15973, 104142, 78253, 47412, 32284, 28453], "top_tokens": [" currency", " currencies", " foreign", " money", "_currency", " dollar", "currency", "\u4eba\u6c11\u5e01", " treasury", " foreigners", " monetary", " Currency"]}, {"layer": 13, "position": -2, "method": "logit", "source_rank": 5389.0, "target_rank": 3249.5, "source_logit": 0.0211181640625, "target_logit": 0.74609375, "top_ids": [3731, 21251, 3776, 28453, 603, 5737, 4229, 11413, 2926, 21525, 1417, 1111], "top_tokens": [" Red", "onia", " One", " Currency", "one", " Group", " Ge", " currency", " Dr", " Ring", " Sh", "ule"]}, {"layer": 13, "position": -1, "method": "logit", "source_rank": 7432.5, "target_rank": 189.0, "source_logit": -0.6015625, "target_logit": 4.5, "top_ids": [7179, 11413, 364, 479, 11766, 356, 467, 362, 2932, 28453, 18174, 17692], "top_tokens": [" International", " currency", " '", " G", " Standard", " C", " W", " A", " She", " Currency", "oret", " dollar"]}, {"layer": 13, "position": -2, "method": "j", "source_rank": 582.0, "target_rank": 142.0, "source_logit": 14.125, "target_logit": 15.9375, "top_ids": [11413, 35602, 13867, 3146, 5313, 29970, 3283, 5837, 7214, 28453, 7576, 11192], "top_tokens": [" currency", " currencies", " county", " country", " national", " tourist", " city", " countries", " foreign", " Currency", " prices", " dollars"]}, {"layer": 13, "position": -1, "method": "j", "source_rank": 909.0, "target_rank": 20.5, "source_logit": 12.4375, "target_logit": 17.0, "top_ids": [11413, 35602, 37198, 7214, 17672, 6489, 7513, 94936, 5837, 17692, 7855, 32488], "top_tokens": [" currency", " currencies", " Commonwealth", " foreign", " euro", " international", " European", " denomination", " countries", " dollar", " British", " euros"]}, {"layer": 17, "position": -2, "method": "logit", "source_rank": 4138.5, "target_rank": 411.0, "source_logit": 0.171875, "target_logit": 3.421875, "top_ids": [4787, 279, 4229, 3693, 89478, 11, 5434, 2598, 271, 220, 15503, 1083], "top_tokens": [" North", " the", " Ge", " American", "\u2026\n", ",", "\u2026\n\n", " called", "\n\n", " ", " primarily", " also"]}, {"layer": 17, "position": -1, "method": "logit", "source_rank": 9022.0, "target_rank": 336.0, "source_logit": -1.2578125, "target_logit": 3.6875, "top_ids": [1852, 3693, 4102, 479, 271, 5938, 4787, 362, 3639, 220, 7179, 4882], "top_tokens": [" same", " American", "\u00a0", " G", "\n\n", " German", " North", " A", " United", " ", " International", " South"]}, {"layer": 17, "position": -2, "method": "j", "source_rank": 1178.0, "target_rank": 669.0, "source_logit": 12.0625, "target_logit": 13.1875, "top_ids": [11413, 35602, 104142, 28453, 103408, 32488, 113883, 106333, 90475, 101406, 26321, 77186], "top_tokens": [" currency", " currencies", "\u4eba\u6c11\u5e01", " Currency", "\u6b27\u5143", " euros", "\u672c\u56fd", "\u5916\u6c47", "(currency", "\u8d27\u5e01", "Currency", ".Currency"]}, {"layer": 17, "position": -1, "method": "j", "source_rank": 2694.0, "target_rank": 594.0, "source_logit": 5.4375, "target_logit": 8.5625, "top_ids": [104142, 101237, 106333, 116866, 109442, 112397, 110065, 103408, 101406, 35602, 11413, 113883], "top_tokens": ["\u4eba\u6c11\u5e01", "\u7f8e\u5143", "\u5916\u6c47", "\u76f4\u8f96\u5e02", "\u8d22\u653f\u90e8", "\u56fd\u5bb6\u6807\u51c6", "\u6c47\u7387", "\u6b27\u5143", "\u8d27\u5e01", " currencies", " currency", "\u672c\u56fd"]}, {"layer": 21, "position": -2, "method": "logit", "source_rank": 18.5, "target_rank": 180.0, "source_logit": 16.25, "target_logit": 8.875, "top_ids": [11413, 104142, 11192, 2598, 17692, 28453, 80189, 58829, 77186, 35602, 34710, 32488], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " dollars", " called", " dollar", " Currency", " Dollars", " Indonesian", ".Currency", " currencies", "_currency", " euros"]}, {"layer": 21, "position": -1, "method": "logit", "source_rank": 4.0, "target_rank": 152.5, "source_logit": 21.25, "target_logit": 8.625, "top_ids": [58829, 104142, 11413, 23968, 17692, 28453, 25662, 23234, 5313, 3639, 3220, 111348], "top_tokens": [" Indonesian", "\u4eba\u6c11\u5e01", " currency", " Indonesia", " dollar", " Currency", " Netherlands", " Dutch", " national", " United", " money", "\u5370\u5c3c"]}, {"layer": 21, "position": -2, "method": "j", "source_rank": 28.0, "target_rank": 204.0, "source_logit": 27.375, "target_logit": 17.0, "top_ids": [11413, 104142, 35602, 11192, 28453, 17692, 80189, 77186, 101406, 101237, 34710, 32488], "top_tokens": [" currency", "\u4eba\u6c11\u5e01", " currencies", " dollars", " Currency", " dollar", " Dollars", ".Currency", "\u8d27\u5e01", "\u7f8e\u5143", "_currency", " euros"]}, {"layer": 21, "position": -1, "method": "j", "source_rank": 3.5, "target_rank": 166.5, "source_logit": 30.0, "target_logit": 14.0625, "top_ids": [104142, 58829, 23968, 11413, 35602, 111348, 110065, 28453, 17692, 101406, 101237, 77186], "top_tokens": ["\u4eba\u6c11\u5e01", " Indonesian", " Indonesia", " currency", " currencies", "\u5370\u5c3c", "\u6c47\u7387", " Currency", " dollar", "\u8d27\u5e01", "\u7f8e\u5143", ".Currency"]}, {"layer": 27, "position": -2, "method": "logit", "source_rank": 9.5, "target_rank": 889.0, "source_logit": 20.5, "target_logit": 10.875, "top_ids": [279, 3881, 2598, 58829, 220, 2274, 111348, 7855, 1083, 23968, 431, 15503], "top_tokens": [" the", " known", " called", " Indonesian", " ", " US", "\u5370\u5c3c", " British", " also", " Indonesia", " R", " primarily"]}, {"layer": 27, "position": -1, "method": "logit", "source_rank": 2.0, "target_rank": 711.0, "source_logit": 25.125, "target_logit": 10.3125, "top_ids": [58829, 23968, 431, 7748, 435, 111348, 2263, 17672, 11122, 2274, 7855, 20539], "top_tokens": [" Indonesian", " Indonesia", " R", " Indian", " r", "\u5370\u5c3c", " Ind", " euro", " ru", " US", " British", " Singapore"]}], "source_token": 23968, "target_token": 6323, "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "input_tokens": ["Fact", ":", " The", " currency", " used", " in", " the", " country", " containing", " Band", "ung", " is", " the"], "seconds": 1.1591262500733137}
{"run_id": "eval-32/baseline/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "baseline", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9693230390548706, "target_logp": -11.671483993530273, "log_odds": -9.702160954475403, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.04021386057138443, -1.9125865697860718, -0.015117242932319641, -0.001405324088409543]}, "target": {"ids": [10769, 57340], "log_probs": [-10.168066024780273, -1.5034180879592896]}}, "score_trace": {"source": [], "target": []}, "output": " Indonesian Rupiah.\n\nFact: The capital of the country", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 6722, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [], "first_top_ids": [58829, 23968, 7748, 431, 435, 111348, 2263, 17672, 11122, 2274], "first_top_logits": [29.25, 25.125, 24.75, 24.75, 23.125, 22.25, 21.625, 20.875, 20.875, 20.625], "source_correct": true, "target_correct": false, "seconds": 0.523261624854058, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/noop/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "noop", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -1.9693230390548706, "target_logp": -11.671483993530273, "log_odds": -9.702160954475403, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.04021386057138443, -1.9125865697860718, -0.015117242932319641, -0.001405324088409543]}, "target": {"ids": [10769, 57340], "log_probs": [-10.168066024780273, -1.5034180879592896]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The capital of the country", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 6722, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "actual_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "relative_norm": [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 23968, 7748, 431, 435, 111348, 2263, 17672, 11122, 2274], "first_top_logits": [29.25, 25.125, 24.75, 24.75, 23.125, 22.25, 21.625, 20.875, 20.875, 20.625], "source_correct": true, "target_correct": false, "seconds": 0.4321485832333565, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/targeted/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "targeted", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -34.075843811035156, "target_logp": -2.0741209983825684, "log_odds": 32.00172281265259, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-32.139678955078125, -1.9126824140548706, -0.021881816908717155, -0.001602675998583436]}, "target": {"ids": [10769, 57340], "log_probs": [-0.2677978575229645, -1.8063231706619263]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 291.4360656738281, 409.5174255371094, 594.5003662109375, 859.2124633789062]], "actual_norm": [[238.7390899658203, 242.70455932617188, 180.9784393310547, 99.592529296875, 102.36357879638672, 101.82274627685547, 97.99144744873047, 196.9324951171875, 77.5084457397461, 291.4548034667969, 409.4925842285156, 594.4140625, 859.2308349609375]], "relative_norm": [[0.015094874426722527, 0.30554232001304626, 0.15658427774906158, 0.08772774040699005, 0.096381276845932, 0.09418239444494247, 0.09437219798564911, 0.19852443039417267, 0.08080793172121048, 0.28204673528671265, 0.43693286180496216, 0.5423982739448547, 0.8368577361106873]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 240.63101196289062, 181.208984375, 101.20629119873047, 102.73998260498047, 103.9610824584961, 98.55260467529297, 200.93511962890625, 76.83016204833984, 292.01654052734375, 403.99163818359375, 596.6821899414062, 859.673828125]], "actual_norm": [[238.7390899658203, 240.60694885253906, 181.2129364013672, 101.26277160644531, 102.70323944091797, 103.98722076416016, 98.56416320800781, 200.9393768310547, 76.86133575439453, 291.9989929199219, 404.0056457519531, 596.631103515625, 859.7125244140625]], "relative_norm": [[0.015094874426722527, 0.30366262793540955, 0.1564352661371231, 0.08930551260709763, 0.09677053987979889, 0.09619168937206268, 0.09500481933355331, 0.20246568322181702, 0.07992532104253769, 0.28100425004959106, 0.4306011497974396, 0.5494723916053772, 0.8377004861831665]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Japanese Yen.  What is the currency used in the", "output_ids": [10769, 809, 268, 13, 220, 3555, 374, 279, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88668823242188, 78.78361511230469, 291.0080871582031, 398.1044616699219, 568.9503173828125, 845.1637573242188]], "actual_norm": [[238.7390899658203, 238.94064331054688, 178.7091827392578, 99.06291198730469, 105.41584014892578, 101.10234069824219, 99.30131530761719, 202.90626525878906, 78.79095458984375, 291.02801513671875, 398.0445251464844, 568.9472045898438, 845.0635375976562]], "relative_norm": [[0.015094874426722527, 0.30528324842453003, 0.1546899825334549, 0.08752020448446274, 0.09913046658039093, 0.09368909895420074, 0.09586968272924423, 0.20392541587352753, 0.08197645097970963, 0.2799966633319855, 0.4237057864665985, 0.5212541222572327, 0.8227953314781189]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [10769, 6323, 809, 57340, 2274, 3639, 619, 7855, 17692, 547], "first_top_logits": [31.75, 30.375, 27.375, 25.375, 25.125, 24.875, 24.625, 24.125, 23.25, 21.875], "source_correct": false, "target_correct": true, "seconds": 0.46691520791500807, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/unrelated/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "unrelated", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -5.99426794052124, "target_logp": -15.743566513061523, "log_odds": -9.749298572540283, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-4.268115997314453, -1.7035911083221436, -0.021302469074726105, -0.0012580584734678268]}, "target": {"ids": [10769, 57340], "log_probs": [-14.141007423400879, -1.6025594472885132]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98483276367188, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51144409179688, 291.4360656738281, 409.5174255371094, 594.5003662109375, 859.2124633789062]], "actual_norm": [[238.64486694335938, 242.7481689453125, 180.9066925048828, 99.57714080810547, 102.39144134521484, 101.81426239013672, 97.92858123779297, 196.97702026367188, 77.55121612548828, 291.5070495605469, 409.5220031738281, 594.556396484375, 859.1492309570312]], "relative_norm": [[0.015088916756212711, 0.3055972456932068, 0.15652219951152802, 0.08771418035030365, 0.09640751779079437, 0.09417454898357391, 0.0943116545677185, 0.19856931269168854, 0.08085251599550247, 0.2820972800254822, 0.4369642734527588, 0.5425281524658203, 0.8367782831192017]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 240.6310272216797, 181.208984375, 101.20629119873047, 102.73998260498047, 103.9610824584961, 98.5526123046875, 200.93511962890625, 76.83015441894531, 292.01654052734375, 403.99163818359375, 596.6821899414062, 859.673828125]], "actual_norm": [[238.64486694335938, 240.663330078125, 181.23175048828125, 101.17539978027344, 102.79727172851562, 103.97574615478516, 98.54431915283203, 200.9502410888672, 76.84894561767578, 292.0298156738281, 404.0358581542969, 596.6934204101562, 859.7026977539062]], "relative_norm": [[0.015088916756212711, 0.30373379588127136, 0.1564515084028244, 0.08922845870256424, 0.09685914218425751, 0.09618107229471207, 0.09498569369316101, 0.2024766206741333, 0.07991243153810501, 0.2810339033603668, 0.4306333661079407, 0.5495297908782959, 0.8376908898353577]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " table.  What is the currency of the country containing Band", "output_ids": [1965, 13, 220, 3555, 374, 279, 11413, 315, 279, 3146, 8482, 16936], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88670349121094, 78.78361511230469, 291.0080871582031, 398.1044616699219, 568.9503784179688, 845.1637573242188]], "actual_norm": [[238.64486694335938, 238.9307861328125, 178.7126922607422, 99.0480728149414, 105.37840270996094, 101.12916564941406, 99.30152893066406, 202.8852996826172, 78.76947784423828, 291.03582763671875, 398.134765625, 568.996337890625, 845.0767211914062]], "relative_norm": [[0.015088916756212711, 0.3052706718444824, 0.15469300746917725, 0.08750709146261215, 0.09909526258707047, 0.09371395409107208, 0.09586988389492035, 0.20390434563159943, 0.08195410668849945, 0.2800041735172272, 0.42380183935165405, 0.5212991237640381, 0.8228081464767456]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [1965, 58829, 6633, 23968, 7748, 220, 3146, 5313, 134597, 13], "first_top_logits": [27.125, 22.875, 20.0, 19.375, 19.0, 18.75, 18.5, 18.125, 18.125, 18.0], "source_correct": false, "target_correct": false, "seconds": 0.43371149990707636, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/negative/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "negative", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -3.1711840629577637, "target_logp": -42.380985260009766, "log_odds": -39.209801197052, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-1.137099027633667, -2.0196547508239746, -0.012826795689761639, -0.0016035090666264296]}, "target": {"ids": [10769, 57340], "log_probs": [-40.57457733154297, -1.8064093589782715]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 291.4360656738281, 409.5174255371094, 594.5003662109375, 859.2124633789062]], "actual_norm": [[238.76498413085938, 242.70928955078125, 180.94493103027344, 99.6073226928711, 102.33980560302734, 101.7933349609375, 97.9751205444336, 196.9191436767578, 77.49484252929688, 291.4145202636719, 409.5401306152344, 594.5711669921875, 859.297119140625]], "relative_norm": [[0.0150965116918087, 0.3055482804775238, 0.15655529499053955, 0.08774077147245407, 0.09635889530181885, 0.0941551923751831, 0.094356469810009, 0.19851097464561462, 0.08079374581575394, 0.28200775384902954, 0.4369836151599884, 0.5425416231155396, 0.8369223475456238]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 240.63101196289062, 181.208984375, 101.20629119873047, 102.73998260498047, 103.9610824584961, 98.55260467529297, 200.93511962890625, 76.83016204833984, 292.01654052734375, 403.99163818359375, 596.6821899414062, 859.673828125]], "actual_norm": [[238.76498413085938, 240.63499450683594, 181.24635314941406, 101.18163299560547, 102.75397491455078, 103.97888946533203, 98.57762145996094, 200.98440551757812, 76.85406494140625, 292.0409240722656, 404.01336669921875, 596.7247924804688, 859.6436157226562]], "relative_norm": [[0.0150965116918087, 0.30369803309440613, 0.1564641147851944, 0.08923395723104477, 0.09681834280490875, 0.09618398547172546, 0.09501779824495316, 0.20251105725765228, 0.07991775870323181, 0.281044602394104, 0.4306093752384186, 0.549558699131012, 0.8376333713531494]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesia Rupiah.\n\nFact: The currency used in the", "output_ids": [23968, 431, 454, 19310, 382, 17417, 25, 576, 11413, 1483, 304, 279], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.6678009033203, 238.97181701660156, 178.6842803955078, 99.06648254394531, 105.40560913085938, 101.10248565673828, 99.27229309082031, 202.88668823242188, 78.78361511230469, 291.0080871582031, 398.1044616699219, 568.9503173828125, 845.1637573242188]], "actual_norm": [[238.76498413085938, 238.98239135742188, 178.67642211914062, 99.09142303466797, 105.44664001464844, 101.12164306640625, 99.24496459960938, 202.9426727294922, 78.762939453125, 290.9903564453125, 398.1314697265625, 569.009033203125, 845.1674194335938]], "relative_norm": [[0.0150965116918087, 0.305336594581604, 0.154661625623703, 0.08754539489746094, 0.09915943443775177, 0.09370698779821396, 0.0958152785897255, 0.20396201312541962, 0.08194729685783386, 0.27996042370796204, 0.4237983226776123, 0.5213107466697693, 0.8228964805603027]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [23968, 58829, 84378, 1257, 111348, 2263, 7748, 20263, 1425, 11413], "first_top_logits": [29.125, 28.5, 20.625, 18.625, 17.625, 17.25, 16.375, 16.25, 16.25, 15.6875], "source_correct": true, "target_correct": false, "seconds": 0.43231487507000566, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/101", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 101, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.216224193572998, "target_logp": -10.197051048278809, "log_odds": -7.9808268547058105, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.2866879999637604, -1.9124962091445923, -0.015913859009742737, -0.0011261318577453494]}, "target": {"ids": [10769, 57340], "log_probs": [-8.594987869262695, -1.6020629405975342]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98483276367188, 99.60225677490234, 102.37149047851562, 101.80586242675781, 97.97480773925781, 196.9394989013672, 77.51143646240234, 291.43603515625, 409.51739501953125, 594.5003662109375, 859.21240234375]], "actual_norm": [[238.40589904785156, 242.71466064453125, 180.92703247070312, 99.69169616699219, 102.40999603271484, 101.88557434082031, 98.0350341796875, 196.90589904785156, 77.48518371582031, 291.4766845703125, 409.6417541503906, 594.5142211914062, 859.2151489257812]], "relative_norm": [[0.015073807910084724, 0.3055550456047058, 0.15653979778289795, 0.08781509101390839, 0.09642498195171356, 0.09424050897359848, 0.09441417455673218, 0.19849762320518494, 0.0807836726307869, 0.2820678949356079, 0.43709203600883484, 0.5424896478652954, 0.8368424773216248]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 240.63099670410156, 181.20896911621094, 101.20629119873047, 102.73997497558594, 103.9610824584961, 98.55260467529297, 200.93511962890625, 76.83015441894531, 292.01654052734375, 403.99163818359375, 596.68212890625, 859.6737670898438]], "actual_norm": [[238.40589904785156, 240.66688537597656, 181.1939239501953, 101.24391174316406, 102.7900161743164, 104.0202407836914, 98.59292602539062, 200.99192810058594, 76.80937194824219, 292.0643005371094, 404.0220642089844, 596.7391357421875, 859.5731811523438]], "relative_norm": [[0.015073807910084724, 0.3037382662296295, 0.15641885995864868, 0.08928888291120529, 0.09685230255126953, 0.09622223675251007, 0.09503254294395447, 0.20251862704753876, 0.0798712819814682, 0.2810671031475067, 0.43061864376068115, 0.5495718717575073, 0.8375647068023682]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The capital of the country", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 6722, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 238.9718017578125, 178.68426513671875, 99.06647491455078, 105.40560913085938, 101.10248565673828, 99.27228546142578, 202.8866729736328, 78.78361511230469, 291.0080871582031, 398.1044616699219, 568.9503173828125, 845.1636962890625]], "actual_norm": [[238.40589904785156, 238.9921112060547, 178.6936798095703, 99.10761260986328, 105.48545837402344, 101.21173095703125, 99.34675598144531, 202.90170288085938, 78.71041870117188, 291.0474853515625, 398.1371154785156, 568.8609008789062, 845.2769775390625]], "relative_norm": [[0.015073807910084724, 0.3053490221500397, 0.15467655658721924, 0.08755969256162643, 0.0991959348320961, 0.09379047155380249, 0.09591355174779892, 0.20392084121704102, 0.08189265429973602, 0.28001537919044495, 0.4238043427467346, 0.5211750268936157, 0.8230031728744507]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 7748, 19461, 431, 17672, 17692, 14533, 2263, 11413, 2274], "first_top_logits": [23.75, 21.625, 20.75, 20.125, 19.875, 19.625, 19.5, 19.5, 19.0, 18.875], "source_correct": true, "target_correct": false, "seconds": 0.4193478748202324, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/202", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 202, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.1013755798339844, "target_logp": -8.74711799621582, "log_odds": -6.645742416381836, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.5779920220375061, -1.5044254064559937, -0.017679952085018158, -0.001278179232031107]}, "target": {"ids": [10769, 57340], "log_probs": [-6.9403395652771, -1.8067784309387207]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66781616210938, 242.7344512939453, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587768554688, 97.97481536865234, 196.93951416015625, 77.51144409179688, 291.4360656738281, 409.5174255371094, 594.5003662109375, 859.2125244140625]], "actual_norm": [[238.5471649169922, 242.68341064453125, 180.9674530029297, 99.68280792236328, 102.44883728027344, 101.87799835205078, 97.96182250976562, 196.9371337890625, 77.51385498046875, 291.35552978515625, 409.52197265625, 594.5054321289062, 859.1834716796875]], "relative_norm": [[0.015082739293575287, 0.30551570653915405, 0.1565747708082199, 0.08780726045370102, 0.09646155685186386, 0.09423349797725677, 0.09434366226196289, 0.19852910935878754, 0.08081356436014175, 0.2819506525993347, 0.436964213848114, 0.5424816012382507, 0.8368116021156311]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66781616210938, 240.6310272216797, 181.20899963378906, 101.206298828125, 102.739990234375, 103.96109008789062, 98.55260467529297, 200.9351348876953, 76.83016967773438, 292.01654052734375, 403.99163818359375, 596.6822509765625, 859.673828125]], "actual_norm": [[238.5471649169922, 240.66160583496094, 181.20877075195312, 101.24750518798828, 102.87300872802734, 103.94770050048828, 98.54735565185547, 200.9149169921875, 76.87257385253906, 291.935302734375, 403.99285888671875, 596.6863403320312, 859.6249389648438]], "relative_norm": [[0.015082739293575287, 0.30373162031173706, 0.1564316749572754, 0.08929204940795898, 0.09693050384521484, 0.09615512937307358, 0.0949886217713356, 0.2024410367012024, 0.07993700355291367, 0.28094297647476196, 0.4305875301361084, 0.5495232939720154, 0.8376151323318481]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The capital of the country", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 6722, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66781616210938, 238.97181701660156, 178.6842803955078, 99.06649017333984, 105.40560913085938, 101.10249328613281, 99.27229309082031, 202.88670349121094, 78.78361511230469, 291.00811767578125, 398.1044921875, 568.9503173828125, 845.1637573242188]], "actual_norm": [[238.5471649169922, 238.9964141845703, 178.61512756347656, 99.02792358398438, 105.49221801757812, 101.14620971679688, 99.22235870361328, 202.85032653808594, 78.80096435546875, 290.98614501953125, 398.1216125488281, 569.00927734375, 845.1807250976562]], "relative_norm": [[0.015082739293575287, 0.305354505777359, 0.1546085625886917, 0.0874892920255661, 0.09920229017734528, 0.09372974932193756, 0.09579344838857651, 0.20386919379234314, 0.08198685944080353, 0.2799563705921173, 0.42378783226013184, 0.5213109850883484, 0.822909414768219]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 13, 2274, 431, 19461, 23968, 435, 3639, 17672, 1532], "first_top_logits": [21.375, 19.75, 19.125, 18.5, 18.375, 18.25, 18.25, 18.125, 18.125, 17.75], "source_correct": true, "target_correct": false, "seconds": 0.4538183747790754, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}
{"run_id": "eval-32/random/{\"layer\": 21, \"positions\": \"all\", \"strength\": 4}/303", "case": {"id": "eval-32", "split": "eval", "country": "Indonesia", "currency": "Indonesian rupiah", "aliases": ["rupiah", "IDR"], "city": "Bandung", "target_country": "Japan", "target_currency": "Japanese yen", "target_aliases": ["yen", "JPY"], "prompt": "Fact: The currency used in the country containing Bandung is the", "source_continuation": " Indonesian rupiah", "target_continuation": " Japanese yen"}, "condition": "random", "config": {"layer": 21, "positions": "all", "strength": 4}, "seed": 303, "generation_seed": 20260907, "sampling": "greedy", "max_new_tokens": 12, "source_logp": -2.385868549346924, "target_logp": -12.662958145141602, "log_odds": -10.277089595794678, "token_scores": {"source": {"ids": [58829, 435, 454, 19310], "log_probs": [-0.018156517297029495, -2.3520326614379883, -0.014099744148552418, -0.0015795861836522818]}, "target": {"ids": [10769, 57340], "log_probs": [-10.643678665161133, -2.0192790031433105]}}, "score_trace": {"source": [{"sequence_length": 17, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 242.73443603515625, 180.98484802246094, 99.60226440429688, 102.37149810791016, 101.80587005615234, 97.97480773925781, 196.93951416015625, 77.51143646240234, 291.43603515625, 409.51739501953125, 594.5003662109375, 859.21240234375]], "actual_norm": [[238.41529846191406, 242.6988067626953, 180.9642791748047, 99.620361328125, 102.40231323242188, 101.79595184326172, 97.93122100830078, 196.9240264892578, 77.58275604248047, 291.40966796875, 409.4162902832031, 594.55615234375, 859.168212890625]], "relative_norm": [[0.015074402093887329, 0.30553507804870605, 0.15657202899456024, 0.08775225281715393, 0.09641774743795395, 0.0941576138138771, 0.09431419521570206, 0.1985158920288086, 0.0808854028582573, 0.2820030450820923, 0.43685147166252136, 0.5425279140472412, 0.836796760559082]], "device": "mps:0", "dtype": "torch.bfloat16"}], "target": [{"sequence_length": 15, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 240.63099670410156, 181.208984375, 101.20628356933594, 102.73997497558594, 103.9610824584961, 98.55260467529297, 200.93511962890625, 76.83015441894531, 292.01654052734375, 403.9916076660156, 596.68212890625, 859.6737670898438]], "actual_norm": [[238.41529846191406, 240.6260986328125, 181.2045135498047, 101.21265411376953, 102.7547607421875, 103.95913696289062, 98.5118179321289, 200.87818908691406, 76.89024353027344, 292.0459289550781, 403.9632263183594, 596.651611328125, 859.5922241210938]], "relative_norm": [[0.015074402093887329, 0.30368679761886597, 0.1564279943704605, 0.08926131576299667, 0.09681908786296844, 0.09616570919752121, 0.09495436400175095, 0.20240402221679688, 0.07995537668466568, 0.2810494303703308, 0.43055593967437744, 0.549491286277771, 0.8375832438468933]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "output": " Indonesian Rupiah.\n\nFact: The capital of the country", "output_ids": [58829, 431, 454, 19310, 382, 17417, 25, 576, 6722, 315, 279, 3146], "input_ids": [17417, 25, 576, 11413, 1483, 304, 279, 3146, 8482, 16936, 2185, 374, 279], "generation_trace": [{"sequence_length": 13, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], "requested_norm": [[238.66778564453125, 238.9718017578125, 178.68426513671875, 99.06647491455078, 105.40560913085938, 101.10247802734375, 99.27229309082031, 202.88668823242188, 78.78361511230469, 291.0080871582031, 398.1044616699219, 568.9503173828125, 845.1637573242188]], "actual_norm": [[238.41529846191406, 238.98301696777344, 178.67726135253906, 99.08872985839844, 105.41427612304688, 101.12664794921875, 99.25846099853516, 202.906494140625, 78.79971313476562, 290.96923828125, 398.0687255859375, 569.0228881835938, 845.1727294921875]], "relative_norm": [[0.015074402093887329, 0.30533739924430847, 0.1546623408794403, 0.08754301071166992, 0.09912899881601334, 0.09371162205934525, 0.09582830965518951, 0.20392565429210663, 0.08198556303977966, 0.27994009852409363, 0.4237315356731415, 0.5213234424591064, 0.8229016661643982]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "first_top_ids": [58829, 23968, 7748, 2274, 111348, 4882, 17692, 3639, 23234, 431], "first_top_logits": [27.625, 23.25, 21.75, 19.875, 19.625, 19.625, 19.375, 19.375, 18.875, 18.75], "source_correct": true, "target_correct": false, "seconds": 0.43245754204690456, "memory": {"max_rss_bytes": 1737801728, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 4020977664}}