{"key": "eval-01-main-baseline", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.37876357006779937, -6.630358451323551e-07], "logp": -0.3787642331036445, "cache_id": 2, "trace": []}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.1912635700678, -0.00011730669405680051], "logp": -21.19138087676186, "cache_id": 3, "trace": []}, "log_odds": -20.812616643658213, "first_log_odds": -20.8125, "later_log_odds": -0.00011664365821317801, "first_differing_log_odds": -20.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-baseline.npz", "array_sha256": "0453ab5b930ee20d8b37edd510b9efecdf88a1e6e0556ade0b3ec7f1e6e66ad4", "seconds": 0.6656051250174642, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456354816, "mps_driver_bytes": 3976937472}}
{"key": "eval-01-main-noop", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.446002006530762, "match_error": -9.446002006530762, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 4, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.37876357006779937, -6.630358451323551e-07], "logp": -0.3787642331036445, "cache_id": 5, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.446002006530762, "match_error": -9.446002006530762, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.1912635700678, -0.00011730669405680051], "logp": -21.19138087676186, "cache_id": 6, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.446002006530762, "match_error": -9.446002006530762, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.812616643658213, "first_log_odds": -20.8125, "later_log_odds": -0.00011664365821317801, "first_differing_log_odds": -20.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-noop.npz", "array_sha256": "0e0fcf8615824459498797faaf60e3d9efe070143792b3a9be0dd461a18801ae", "seconds": 0.34750933293253183, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456387584, "mps_driver_bytes": 3976937472}}
{"key": "eval-01-main-j", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.439163208007812, "actual_norm": 9.446002006530762, "relative_norm": 0.04999962745804996, "target_actual_norm": 9.446002006530762, "match_error": 0.0, "scalar": 0.17027775195379694, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 7, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.3343864750158404, -6.708824925599188e-07], "logp": -0.33438714589833296, "cache_id": 8, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.439163208007812, "actual_norm": 9.446002006530762, "relative_norm": 0.04999962745804996, "target_actual_norm": 9.446002006530762, "match_error": 0.0, "scalar": 0.17027775195379694, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-20.58438647501584, -5.046588748307587e-05], "logp": -20.584436940903323, "cache_id": 9, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.439163208007812, "actual_norm": 9.446002006530762, "relative_norm": 0.04999962745804996, "target_actual_norm": 9.446002006530762, "match_error": 0.0, "scalar": 0.17027775195379694, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.25004979500499, "first_log_odds": -20.25, "later_log_odds": -4.9795004990471625e-05, "first_differing_log_odds": -20.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-j.npz", "array_sha256": "f6c0331527e2a3f7b17cce4a9ac4b887572bf81d313a21dbb6c6a0ecf927b1ea", "seconds": 0.3279812908731401, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456387584, "mps_driver_bytes": 3976937472}}
{"key": "eval-01-main-opposite", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.430542945861816, "actual_norm": 9.442214012145996, "relative_norm": 0.049979576826267275, "target_actual_norm": 9.446002006530762, "match_error": -0.003787994384765625, "scalar": 9.430543575817865, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 10, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.33077504427590526, -6.348899124947465e-07], "logp": -0.33077567916581774, "cache_id": 11, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.430542945861816, "actual_norm": 9.442214012145996, "relative_norm": 0.049979576826267275, "target_actual_norm": 9.446002006530762, "match_error": -0.003787994384765625, "scalar": 9.430543575817865, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.830775044275907, -0.00021687192433802565], "logp": -21.830991916200244, "cache_id": 12, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.430542945861816, "actual_norm": 9.442214012145996, "relative_norm": 0.049979576826267275, "target_actual_norm": 9.446002006530762, "match_error": -0.003787994384765625, "scalar": 9.430543575817865, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.500216237034426, "first_log_odds": -21.5, "later_log_odds": -0.00021623703442585906, "first_differing_log_odds": -21.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-opposite.npz", "array_sha256": "595ac45f75d3935d859607d1d466110261f9db8268e1df182489202683e8057f", "seconds": 0.33003999991342425, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456387584, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-unrelated", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.457316398620605, "actual_norm": 9.446029663085938, "relative_norm": 0.04999977384987315, "target_actual_norm": 9.446002006530762, "match_error": 2.765655517578125e-05, "scalar": 9.457316568870738, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 13, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.35207810028540004, -6.525660114693368e-07], "logp": -0.3520787528514115, "cache_id": 14, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.457316398620605, "actual_norm": 9.446029663085938, "relative_norm": 0.04999977384987315, "target_actual_norm": 9.446002006530762, "match_error": 2.765655517578125e-05, "scalar": 9.457316568870738, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.1645781002854, -0.0002150621410880642], "logp": -21.164793162426488, "cache_id": 15, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.457316398620605, "actual_norm": 9.446029663085938, "relative_norm": 0.04999977384987315, "target_actual_norm": 9.446002006530762, "match_error": 2.765655517578125e-05, "scalar": 9.457316568870738, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.812714409575076, "first_log_odds": -20.8125, "later_log_odds": -0.00021440957507579128, "first_differing_log_odds": -20.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-unrelated.npz", "array_sha256": "f0eb6cda8e2d076dc21352a811a6908e79d2b198a4c2988961cb1d5af8bf3b83", "seconds": 0.3336305422708392, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456420352, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-ordinary", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.452125549316406, "actual_norm": 9.445990562438965, "relative_norm": 0.04999956688212312, "target_actual_norm": 9.446002006530762, "match_error": -1.1444091796875e-05, "scalar": 9.45212772499508, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 16, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.37875162198748447, -6.599387827982945e-07], "logp": -0.37875228192626725, "cache_id": 17, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.452125549316406, "actual_norm": 9.445990562438965, "relative_norm": 0.04999956688212312, "target_actual_norm": 9.446002006530762, "match_error": -1.1444091796875e-05, "scalar": 9.45212772499508, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.066251621987483, -0.00011675285079734029], "logp": -21.06636837483828, "cache_id": 18, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.452125549316406, "actual_norm": 9.445990562438965, "relative_norm": 0.04999956688212312, "target_actual_norm": 9.446002006530762, "match_error": -1.1444091796875e-05, "scalar": 9.45212772499508, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.687616092912013, "first_log_odds": -20.6875, "later_log_odds": -0.00011609291201253313, "first_differing_log_odds": -20.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-ordinary.npz", "array_sha256": "8f02f809b8570028353cf19f4a35daf81f2d02c8ce960cebe83be0491caf35fd", "seconds": 0.33237366611137986, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-random0", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.432920455932617, "actual_norm": 9.445609092712402, "relative_norm": 0.049997547684561924, "target_actual_norm": 9.446002006530762, "match_error": -0.000392913818359375, "scalar": 9.432921795927541, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 19, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.4329725219617885, -7.421096812339456e-07], "logp": -0.4329732640714697, "cache_id": 20, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.432920455932617, "actual_norm": 9.445609092712402, "relative_norm": 0.049997547684561924, "target_actual_norm": 9.446002006530762, "match_error": -0.000392913818359375, "scalar": 9.432921795927541, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.120472521961787, -0.0001686006465293759], "logp": -21.120641122608315, "cache_id": 21, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.432920455932617, "actual_norm": 9.445609092712402, "relative_norm": 0.049997547684561924, "target_actual_norm": 9.446002006530762, "match_error": -0.000392913818359375, "scalar": 9.432921795927541, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.687667858536845, "first_log_odds": -20.6875, "later_log_odds": -0.000167858536844534, "first_differing_log_odds": -20.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-random0.npz", "array_sha256": "eda5f1d5fc54b065a4c4075a8e58064e53c94153eb24233837e77ce301ba3ba1", "seconds": 0.3297854997217655, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-random1", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.441099166870117, "actual_norm": 9.445900917053223, "relative_norm": 0.04999909237069624, "target_actual_norm": 9.446002006530762, "match_error": -0.0001010894775390625, "scalar": 9.441101431759307, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 22, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.378765474057778, -6.61682988646757e-07], "logp": -0.3787661357407666, "cache_id": 23, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.441099166870117, "actual_norm": 9.445900917053223, "relative_norm": 0.04999909237069624, "target_actual_norm": 9.446002006530762, "match_error": -0.0001010894775390625, "scalar": 9.441101431759307, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.12876547405778, -0.00015049186675861558], "logp": -21.128915965924538, "cache_id": 24, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.441099166870117, "actual_norm": 9.445900917053223, "relative_norm": 0.04999909237069624, "target_actual_norm": 9.446002006530762, "match_error": -0.0001010894775390625, "scalar": 9.441101431759307, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.750149830183773, "first_log_odds": -20.75, "later_log_odds": -0.00014983018377279222, "first_differing_log_odds": -20.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-random1.npz", "array_sha256": "c05c6204ebe95cc74e88b69687c2b7b0b9298eb5a751f4ecb1217e961d8f449b", "seconds": 0.33216500002890825, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-random2", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.449227333068848, "actual_norm": 9.44595718383789, "relative_norm": 0.04999939020233652, "target_actual_norm": 9.446002006530762, "match_error": -4.482269287109375e-05, "scalar": 9.449227017134035, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 25, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.43298529585965284, -7.307330737403423e-07], "logp": -0.43298602659272656, "cache_id": 26, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.449227333068848, "actual_norm": 9.44595718383789, "relative_norm": 0.04999939020233652, "target_actual_norm": 9.446002006530762, "match_error": -4.482269287109375e-05, "scalar": 9.449227017134035, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-21.307985295859652, -0.00011797930236579325], "logp": -21.30810327516202, "cache_id": 27, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.92144775390625, "strength_one_norm": 55.43391799926758, "target_shrink_scale": 0.17027775195379694, "requested_norm": 9.449227333068848, "actual_norm": 9.44595718383789, "relative_norm": 0.04999939020233652, "target_actual_norm": 9.446002006530762, "match_error": -4.482269287109375e-05, "scalar": 9.449227017134035, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.875117248569293, "first_log_odds": -20.875, "later_log_odds": -0.00011724856929262728, "first_differing_log_odds": -20.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-random2.npz", "array_sha256": "eaeb6e373b0eb81e1c119647d33cca7c8256c6585949d0351547a9874e1b300e", "seconds": 0.3393970844335854, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456453120, "mps_driver_bytes": 3985326080}}
{"key": "eval-01-main-bridge", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "India Rupee is used in India, but the question is about Tokyo. Tokyo is in Japan, and the currency used there is the Japanese Yen. The three-letter ISO 4217 code for the Japanese Yen is JPY.<|im_end|>", "output_text": "India Rupee is used in India, but the question is about Tokyo. Tokyo is in Japan, and the currency used there is the Japanese Yen. The three-letter ISO 4217 code for the Japanese Yen is JPY.", "output_ids": [33548, 27914, 61498, 374, 1483, 304, 6747, 11, 714, 279, 3405, 374, 911, 26194, 13, 26194, 374, 304, 6323, 11, 323, 279, 11413, 1483, 1052, 374, 279, 10769, 809, 268, 13, 576, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 10769, 809, 268, 374, 619, 37881, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 1024.2030029296875, 39.48552322387695, 270.3546447753906, 78.30937957763672, 161.1520233154297, 262.9189453125, 211.51675415039062, 335.17962646484375, 41.48118209838867, 32.48482131958008, 6.306441783905029, 165.87107849121094, 149.33624267578125, 4.212922096252441, 41.258697509765625, 56.32947540283203, 14.588085174560547, 93.457763671875, 72.52143096923828, 173.5310516357422, 207.1820831298828, 88.47539520263672, 141.42913818359375, 28.933881759643555, 78.65763092041016, 1408.261474609375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 1024.2835693359375, 39.51523208618164, 270.335205078125, 78.33322143554688, 161.20199584960938, 262.935302734375, 211.53814697265625, 335.142822265625, 41.548377990722656, 32.500091552734375, 6.5008392333984375, 165.89047241210938, 149.3659210205078, 4.319800853729248, 41.32025146484375, 56.365623474121094, 14.610169410705566, 93.4968490600586, 72.55130767822266, 173.494873046875, 207.14663696289062, 88.3357162475586, 141.4431915283203, 28.96887969970703, 78.66455078125, 1408.1944580078125]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.9306935667991638, 0.04089762270450592, 0.26484331488609314, 0.07510987669229507, 0.1568710058927536, 0.25331684947013855, 0.18603134155273438, 0.32572677731513977, 0.03959348797798157, 0.03308093920350075, 0.006686883512884378, 0.1664063185453415, 0.1445096880197525, 0.004168200772255659, 0.03346136212348938, 0.057041168212890625, 0.01648719795048237, 0.10268690437078476, 0.08640759438276291, 0.17189271748065948, 0.2204907089471817, 0.07778435200452805, 0.1362382024526596, 0.030961843207478523, 0.07619304955005646, 1.3198931217193604]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 28, "cached": true, "scoring": {"normalized": "India Rupee is used in India, but the question is about Tokyo. Tokyo is in Japan, and the currency used there is the Japanese Yen. The three-letter ISO 4217 code for the Japanese Yen is JPY.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "India Rupee is used in India, but the question is about Tokyo. Tokyo is in Japan, and the currency used there is the Japanese Yen. The three-letter ISO 4217 code for the Japanese Yen is ", "substantive_prefix_before_code": true, "country_mentions": [{"text": "India", "start": 0, "end": 5}, {"text": "India", "start": 23, "end": 28}, {"text": "Japan", "start": 75, "end": 80}, {"text": "Japanese", "start": 117, "end": 125}, {"text": "Japanese", "start": 170, "end": 178}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-37.98151832626678, -7.531533609414579e-07], "logp": -37.98151907942014, "cache_id": 29, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 1024.2030029296875, 39.48552322387695, 270.3546447753906, 78.30937957763672, 161.1520233154297, 262.9189453125, 211.51675415039062, 335.17962646484375, 41.48118209838867, 32.48482131958008, 6.306441783905029, 165.87107849121094, 149.33624267578125, 4.212922096252441, 41.258697509765625, 56.32947540283203, 14.588085174560547, 93.457763671875, 72.52143096923828, 173.5310516357422, 207.1820831298828, 88.47539520263672, 141.42913818359375, 28.933881759643555, 78.65763092041016, 1408.261474609375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 1024.2835693359375, 39.51523208618164, 270.335205078125, 78.33322143554688, 161.20199584960938, 262.935302734375, 211.53814697265625, 335.142822265625, 41.548377990722656, 32.500091552734375, 6.5008392333984375, 165.89047241210938, 149.3659210205078, 4.319800853729248, 41.32025146484375, 56.365623474121094, 14.610169410705566, 93.4968490600586, 72.55130767822266, 173.494873046875, 207.14663696289062, 88.3357162475586, 141.4431915283203, 28.96887969970703, 78.66455078125, 1408.1944580078125]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.9306935667991638, 0.04089762270450592, 0.26484331488609314, 0.07510987669229507, 0.1568710058927536, 0.25331684947013855, 0.18603134155273438, 0.32572677731513977, 0.03959348797798157, 0.03308093920350075, 0.006686883512884378, 0.1664063185453415, 0.1445096880197525, 0.004168200772255659, 0.03346136212348938, 0.057041168212890625, 0.01648719795048237, 0.10268690437078476, 0.08640759438276291, 0.17189271748065948, 0.2204907089471817, 0.07778435200452805, 0.1362382024526596, 0.030961843207478523, 0.07619304955005646, 1.3198931217193604]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-6.544018326266783, -0.00019041106885911007], "logp": -6.544208737335642, "cache_id": 30, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 1024.2030029296875, 39.48552322387695, 270.3546447753906, 78.30937957763672, 161.1520233154297, 262.9189453125, 211.51675415039062, 335.17962646484375, 41.48118209838867, 32.48482131958008, 6.306441783905029, 165.87107849121094, 149.33624267578125, 4.212922096252441, 41.258697509765625, 56.32947540283203, 14.588085174560547, 93.457763671875, 72.52143096923828, 173.5310516357422, 207.1820831298828, 88.47539520263672, 141.42913818359375, 28.933881759643555, 78.65763092041016, 1408.261474609375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 1024.2835693359375, 39.51523208618164, 270.335205078125, 78.33322143554688, 161.20199584960938, 262.935302734375, 211.53814697265625, 335.142822265625, 41.548377990722656, 32.500091552734375, 6.5008392333984375, 165.89047241210938, 149.3659210205078, 4.319800853729248, 41.32025146484375, 56.365623474121094, 14.610169410705566, 93.4968490600586, 72.55130767822266, 173.494873046875, 207.14663696289062, 88.3357162475586, 141.4431915283203, 28.96887969970703, 78.66455078125, 1408.1944580078125]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.9306935667991638, 0.04089762270450592, 0.26484331488609314, 0.07510987669229507, 0.1568710058927536, 0.25331684947013855, 0.18603134155273438, 0.32572677731513977, 0.03959348797798157, 0.03308093920350075, 0.006686883512884378, 0.1664063185453415, 0.1445096880197525, 0.004168200772255659, 0.03346136212348938, 0.057041168212890625, 0.01648719795048237, 0.10268690437078476, 0.08640759438276291, 0.17189271748065948, 0.2204907089471817, 0.07778435200452805, 0.1362382024526596, 0.030961843207478523, 0.07619304955005646, 1.3198931217193604]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 31.437310342084498, "first_log_odds": 31.437499999999996, "later_log_odds": -0.0001896579154987421, "first_differing_log_odds": 31.437499999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-main-bridge.npz", "array_sha256": "94d59d5d77cc1ef9de204ec6f8bd53fad440b0f0a0e87318cd26673e85d5fa5f", "seconds": 1.9547837916761637, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-baseline", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 31, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.750064122289288, -8.814747590415662e-07], "logp": -9.750065003764048, "cache_id": 32, "trace": []}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.000064122289288, -0.0006520819719405025], "logp": -31.000716204261227, "cache_id": 33, "trace": []}, "log_odds": -21.25065120049718, "first_log_odds": -21.25, "later_log_odds": -0.000651200497181037, "first_differing_log_odds": -21.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-baseline.npz", "array_sha256": "c94ba82db45dacd98feaff9006eb82c3a3dc32c3fe7266c8a5e8568926db12ea", "seconds": 1.208590124733746, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-noop", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.396490097045898, "match_error": -9.396490097045898, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 34, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.750064122289288, -8.814747590415662e-07], "logp": -9.750065003764048, "cache_id": 35, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.396490097045898, "match_error": -9.396490097045898, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.000064122289288, -0.0006520819719405025], "logp": -31.000716204261227, "cache_id": 36, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.396490097045898, "match_error": -9.396490097045898, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.25065120049718, "first_log_odds": -21.25, "later_log_odds": -0.000651200497181037, "first_differing_log_odds": -21.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-noop.npz", "array_sha256": "cee84fc61a850c80274971859e2d979f93029330cd8ca0c07954aa34d404f5d0", "seconds": 1.1863790000788867, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-j", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.392239570617676, "actual_norm": 9.396490097045898, "relative_norm": 0.04999989241806357, "target_actual_norm": 9.396490097045898, "match_error": 0.0, "scalar": 0.16461787370515893, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 37, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.000051199517552, -1.2807682759284833e-06], "logp": -10.000052480285827, "cache_id": 38, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.392239570617676, "actual_norm": 9.396490097045898, "relative_norm": 0.04999989241806357, "target_actual_norm": 9.396490097045898, "match_error": 0.0, "scalar": 0.16461787370515893, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.937551199517554, -0.0003639404698015924], "logp": -30.937915139987354, "cache_id": 39, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.392239570617676, "actual_norm": 9.396490097045898, "relative_norm": 0.04999989241806357, "target_actual_norm": 9.396490097045898, "match_error": 0.0, "scalar": 0.16461787370515893, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.937862659701526, "first_log_odds": -20.9375, "later_log_odds": -0.0003626597015262689, "first_differing_log_odds": -20.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-j.npz", "array_sha256": "258cdda862ff283c78cc898cbe1969750a7010c8a3d9ed2eae079f52b784f726", "seconds": 1.1810011668130755, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-opposite", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.389117240905762, "actual_norm": 9.396053314208984, "relative_norm": 0.04999756824226705, "target_actual_norm": 9.396490097045898, "match_error": -0.0004367828369140625, "scalar": 9.389115036630756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 40, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.500082350052606, -1.2857768659275852e-06], "logp": -9.500083635829473, "cache_id": 41, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.389117240905762, "actual_norm": 9.396053314208984, "relative_norm": 0.04999756824226705, "target_actual_norm": 9.396490097045898, "match_error": -0.0004367828369140625, "scalar": 9.389115036630756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.000082350052605, -0.0006612399405450171], "logp": -31.00074358999315, "cache_id": 42, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.389117240905762, "actual_norm": 9.396053314208984, "relative_norm": 0.04999756824226705, "target_actual_norm": 9.396490097045898, "match_error": -0.0004367828369140625, "scalar": 9.389115036630756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.500659954163677, "first_log_odds": -21.5, "later_log_odds": -0.0006599541636767015, "first_differing_log_odds": -21.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-opposite.npz", "array_sha256": "f5b80bf3f2e3a1428cd0dada6731b7fe739e80b5337d3138f82adb52087d3d1e", "seconds": 1.1856917911209166, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-unrelated", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40408706665039, "actual_norm": 9.396350860595703, "relative_norm": 0.04999915152359568, "target_actual_norm": 9.396490097045898, "match_error": -0.0001392364501953125, "scalar": 9.40408918724279, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 43, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.250039446590083, -1.2828305225453435e-06], "logp": -10.250040729420606, "cache_id": 44, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40408706665039, "actual_norm": 9.396350860595703, "relative_norm": 0.04999915152359568, "target_actual_norm": 9.396490097045898, "match_error": -0.0001392364501953125, "scalar": 9.40408918724279, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.312539446590083, -0.00071514483486446], "logp": -31.313254591424947, "cache_id": 45, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40408706665039, "actual_norm": 9.396350860595703, "relative_norm": 0.04999915152359568, "target_actual_norm": 9.396490097045898, "match_error": -0.0001392364501953125, "scalar": 9.40408918724279, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.06321386200434, "first_log_odds": -21.0625, "later_log_odds": -0.0007138620043392052, "first_differing_log_odds": -21.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-unrelated.npz", "array_sha256": "b513aafb9bf774f6171fbb123d012d07833aecb59f614ba96546d931b853a0f1", "seconds": 1.1803655000403523, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-ordinary", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40279769897461, "actual_norm": 9.396122932434082, "relative_norm": 0.049997938689501, "target_actual_norm": 9.396490097045898, "match_error": -0.00036716461181640625, "scalar": 9.402798775699921, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 46, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.750065602921127, -1.2719031752654947e-06], "logp": -9.750066874824302, "cache_id": 47, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40279769897461, "actual_norm": 9.396122932434082, "relative_norm": 0.049997938689501, "target_actual_norm": 9.396490097045898, "match_error": -0.00036716461181640625, "scalar": 9.402798775699921, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.87506560292113, -0.00045943914453311396], "logp": -30.87552504206566, "cache_id": 48, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40279769897461, "actual_norm": 9.396122932434082, "relative_norm": 0.049997938689501, "target_actual_norm": 9.396490097045898, "match_error": -0.00036716461181640625, "scalar": 9.402798775699921, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.12545816724136, "first_log_odds": -21.125, "later_log_odds": -0.0004581672413586091, "first_differing_log_odds": -21.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-ordinary.npz", "array_sha256": "b5783f0a906dbbcf3defb5d6ce032b10685be1d21cb8c9898beddf91c7e84d64", "seconds": 1.1851399578154087, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-random0", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.397851943969727, "actual_norm": 9.395528793334961, "relative_norm": 0.04999477720146337, "target_actual_norm": 9.396490097045898, "match_error": -0.0009613037109375, "scalar": 9.397852198118926, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 49, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.750064699056113, -1.261372273575446e-06], "logp": -9.750065960428387, "cache_id": 50, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.397851943969727, "actual_norm": 9.395528793334961, "relative_norm": 0.04999477720146337, "target_actual_norm": 9.396490097045898, "match_error": -0.0009613037109375, "scalar": 9.397852198118926, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.18756469905611, -0.000669327350496342], "logp": -31.188234026406608, "cache_id": 51, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.397851943969727, "actual_norm": 9.395528793334961, "relative_norm": 0.04999477720146337, "target_actual_norm": 9.396490097045898, "match_error": -0.0009613037109375, "scalar": 9.397852198118926, "random_seed": 20271008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.43816806597822, "first_log_odds": -21.4375, "later_log_odds": -0.0006680659782212217, "first_differing_log_odds": -21.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-random0.npz", "array_sha256": "aed17759d058cf6137f2d0bb172d1b90bad3d9211fc567a400f4ff47b58bdede", "seconds": 1.184038208797574, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-random1", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.391578674316406, "actual_norm": 9.396479606628418, "relative_norm": 0.04999983659724749, "target_actual_norm": 9.396490097045898, "match_error": -1.049041748046875e-05, "scalar": 9.391579364229983, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 52, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.250039505372548, -1.2555787152392583e-06], "logp": -10.250040760951263, "cache_id": 53, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.391578674316406, "actual_norm": 9.396479606628418, "relative_norm": 0.04999983659724749, "target_actual_norm": 9.396490097045898, "match_error": -1.049041748046875e-05, "scalar": 9.391579364229983, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.31253950537255, -0.0004653486448667248], "logp": -31.313004854017414, "cache_id": 54, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.391578674316406, "actual_norm": 9.396479606628418, "relative_norm": 0.04999983659724749, "target_actual_norm": 9.396490097045898, "match_error": -1.049041748046875e-05, "scalar": 9.391579364229983, "random_seed": 20271009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.06296409306615, "first_log_odds": -21.0625, "later_log_odds": -0.00046409306614947354, "first_differing_log_odds": -21.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-random1.npz", "array_sha256": "4abe6cdbe3ce78f4e1a85e3307fe3d2d97c099838d3551d6fcccc6e91f6af36e", "seconds": 1.1823706668801606, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-random2", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40160846710205, "actual_norm": 9.396471977233887, "relative_norm": 0.04999979600029035, "target_actual_norm": 9.396490097045898, "match_error": -1.811981201171875e-05, "scalar": 9.40160805740993, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 55, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.500082215012418, -1.002377890352875e-06], "logp": -9.500083217390308, "cache_id": 56, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40160846710205, "actual_norm": 9.396471977233887, "relative_norm": 0.04999979600029035, "target_actual_norm": 9.396490097045898, "match_error": -1.811981201171875e-05, "scalar": 9.40160805740993, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.812582215012416, -0.0007322554663308121], "logp": -30.813314470478748, "cache_id": 57, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.93020629882812, "strength_one_norm": 57.05479049682617, "target_shrink_scale": 0.16461787370515893, "requested_norm": 9.40160846710205, "actual_norm": 9.396471977233887, "relative_norm": 0.04999979600029035, "target_actual_norm": 9.396490097045898, "match_error": -1.811981201171875e-05, "scalar": 9.40160805740993, "random_seed": 20271010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.31323125308844, "first_log_odds": -21.3125, "later_log_odds": -0.0007312530884391322, "first_differing_log_odds": -21.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-random2.npz", "array_sha256": "0ca2a08151b34a35520def0dcf4ad66b66bf56f2d3d0fb80675d69079d66fbde", "seconds": 1.1771488329395652, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-01-paraphrase-bridge", "case_id": "eval-01", "city": "Tokyo", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 26194, 374, 279, 10769, 809, 268, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 1028.3094482421875, 36.26524353027344, 107.65953063964844, 383.52734375, 148.4219207763672, 236.5840301513672, 203.44876098632812, 352.6665954589844, 47.126258850097656, 44.155941009521484, 22.725622177124023, 172.694091796875, 11.995611190795898, 74.09344482421875, 81.43805694580078, 91.23380279541016, 188.93869018554688, 221.7884979248047, 114.60735321044922, 145.31700134277344, 54.586788177490234, 77.51548767089844, 1198.9732666015625]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 1028.4520263671875, 36.20345687866211, 107.63091278076172, 383.531982421875, 148.4624481201172, 236.61932373046875, 203.42941284179688, 352.58837890625, 47.14916229248047, 44.177513122558594, 22.762287139892578, 172.73519897460938, 12.057031631469727, 74.107666015625, 81.41692352294922, 91.18428802490234, 188.9134979248047, 221.80252075195312, 114.6185531616211, 145.3230438232422, 54.65401840209961, 77.53550720214844, 1198.92724609375]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.9487011432647705, 0.03420029580593109, 0.09789130091667175, 0.3531387448310852, 0.13462486863136292, 0.22816450893878937, 0.17431728541851044, 0.3309227228164673, 0.04434720054268837, 0.04445922002196312, 0.023049507290124893, 0.1690322905778885, 0.01143491081893444, 0.07690559327602386, 0.09155257046222687, 0.10434038937091827, 0.18355755507946014, 0.23556308448314667, 0.09963761270046234, 0.1373501867055893, 0.05767165869474411, 0.07478716224431992, 1.227910041809082]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 58, "cached": true, "scoring": {"normalized": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Tokyo is the Japanese Yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 34, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-43.5313264689916, -1.4076628984723102e-06], "logp": -43.5313278766545, "cache_id": 59, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 1028.3094482421875, 36.26524353027344, 107.65953063964844, 383.52734375, 148.4219207763672, 236.5840301513672, 203.44876098632812, 352.6665954589844, 47.126258850097656, 44.155941009521484, 22.725622177124023, 172.694091796875, 11.995611190795898, 74.09344482421875, 81.43805694580078, 91.23380279541016, 188.93869018554688, 221.7884979248047, 114.60735321044922, 145.31700134277344, 54.586788177490234, 77.51548767089844, 1198.9732666015625]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 1028.4520263671875, 36.20345687866211, 107.63091278076172, 383.531982421875, 148.4624481201172, 236.61932373046875, 203.42941284179688, 352.58837890625, 47.14916229248047, 44.177513122558594, 22.762287139892578, 172.73519897460938, 12.057031631469727, 74.107666015625, 81.41692352294922, 91.18428802490234, 188.9134979248047, 221.80252075195312, 114.6185531616211, 145.3230438232422, 54.65401840209961, 77.53550720214844, 1198.92724609375]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.9487011432647705, 0.03420029580593109, 0.09789130091667175, 0.3531387448310852, 0.13462486863136292, 0.22816450893878937, 0.17431728541851044, 0.3309227228164673, 0.04434720054268837, 0.04445922002196312, 0.023049507290124893, 0.1690322905778885, 0.01143491081893444, 0.07690559327602386, 0.09155257046222687, 0.10434038937091827, 0.18355755507946014, 0.23556308448314667, 0.09963761270046234, 0.1373501867055893, 0.05767165869474411, 0.07478716224431992, 1.227910041809082]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-18.3750764689916, -0.0003782522736597911], "logp": -18.37545472126526, "cache_id": 60, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 1028.3094482421875, 36.26524353027344, 107.65953063964844, 383.52734375, 148.4219207763672, 236.5840301513672, 203.44876098632812, 352.6665954589844, 47.126258850097656, 44.155941009521484, 22.725622177124023, 172.694091796875, 11.995611190795898, 74.09344482421875, 81.43805694580078, 91.23380279541016, 188.93869018554688, 221.7884979248047, 114.60735321044922, 145.31700134277344, 54.586788177490234, 77.51548767089844, 1198.9732666015625]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 1028.4520263671875, 36.20345687866211, 107.63091278076172, 383.531982421875, 148.4624481201172, 236.61932373046875, 203.42941284179688, 352.58837890625, 47.14916229248047, 44.177513122558594, 22.762287139892578, 172.73519897460938, 12.057031631469727, 74.107666015625, 81.41692352294922, 91.18428802490234, 188.9134979248047, 221.80252075195312, 114.6185531616211, 145.3230438232422, 54.65401840209961, 77.53550720214844, 1198.92724609375]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.9487011432647705, 0.03420029580593109, 0.09789130091667175, 0.3531387448310852, 0.13462486863136292, 0.22816450893878937, 0.17431728541851044, 0.3309227228164673, 0.04434720054268837, 0.04445922002196312, 0.023049507290124893, 0.1690322905778885, 0.01143491081893444, 0.07690559327602386, 0.09155257046222687, 0.10434038937091827, 0.18355755507946014, 0.23556308448314667, 0.09963761270046234, 0.1373501867055893, 0.05767165869474411, 0.07478716224431992, 1.227910041809082]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 25.15587315538924, "first_log_odds": 25.15625, "later_log_odds": -0.0003768446107592638, "first_differing_log_odds": 25.15625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-01-paraphrase-bridge.npz", "array_sha256": "b881221b4b6d736c657a5b1b3afaafdb5f6adce37570456a775c5a2a33f91a3a", "seconds": 1.35538891563192, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-baseline", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 61, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.1572830362019111, -7.829048762134328e-07], "logp": -0.1572838191067873, "cache_id": 62, "trace": []}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-19.28228303620191, -2.4077366662227063e-05], "logp": -19.282307113568574, "cache_id": 63, "trace": []}, "log_odds": -19.125023294461787, "first_log_odds": -19.125, "later_log_odds": -2.3294461787060072e-05, "first_differing_log_odds": -19.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-baseline.npz", "array_sha256": "b9bf8eb35ccb5c81916d270a67cfb97f08441409b5ef93e36e56752a3b3ac245", "seconds": 0.37642495799809694, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-noop", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.25214958190918, "match_error": -9.25214958190918, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 64, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.1572830362019111, -7.829048762134328e-07], "logp": -0.1572838191067873, "cache_id": 65, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.25214958190918, "match_error": -9.25214958190918, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-19.28228303620191, -2.4077366662227063e-05], "logp": -19.282307113568574, "cache_id": 66, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.25214958190918, "match_error": -9.25214958190918, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.125023294461787, "first_log_odds": -19.125, "later_log_odds": -2.3294461787060072e-05, "first_differing_log_odds": -19.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-noop.npz", "array_sha256": "795c36b5bfd777e5295c2233667907870b137710ce632c26ff696bda51e2c153", "seconds": 0.33472029073163867, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-j", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.257231712341309, "actual_norm": 9.25214958190918, "relative_norm": 0.049972549455278834, "target_actual_norm": 9.25214958190918, "match_error": 0.0, "scalar": 0.25950341921004805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 67, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.16506985829822735, -8.601925438217597e-07], "logp": -0.16507071849077118, "cache_id": 68, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.257231712341309, "actual_norm": 9.25214958190918, "relative_norm": 0.049972549455278834, "target_actual_norm": 9.25214958190918, "match_error": 0.0, "scalar": 0.25950341921004805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-17.790069858298228, -6.637224868332712e-06], "logp": -17.790076495523095, "cache_id": 69, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.257231712341309, "actual_norm": 9.25214958190918, "relative_norm": 0.049972549455278834, "target_actual_norm": 9.25214958190918, "match_error": 0.0, "scalar": 0.25950341921004805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.625005777032325, "first_log_odds": -17.625, "later_log_odds": -5.77703232451654e-06, "first_differing_log_odds": -17.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-j.npz", "array_sha256": "74249a6aa44be214e8b1b5517361eedce71c7923cc097b2ec6f7f1ddd72113fe", "seconds": 0.33618120895698667, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-opposite", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.249183654785156, "actual_norm": 9.252118110656738, "relative_norm": 0.049972379473297336, "target_actual_norm": 9.25214958190918, "match_error": -3.147125244140625e-05, "scalar": 9.249184873333434, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 70, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.14334553954855475, -8.429255176174493e-07], "logp": -0.14334638247407236, "cache_id": 71, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.249183654785156, "actual_norm": 9.252118110656738, "relative_norm": 0.049972379473297336, "target_actual_norm": 9.25214958190918, "match_error": -3.147125244140625e-05, "scalar": 9.249184873333434, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-20.018345539548555, -0.00011602648668875191], "logp": -20.018461566035246, "cache_id": 72, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.249183654785156, "actual_norm": 9.252118110656738, "relative_norm": 0.049972379473297336, "target_actual_norm": 9.25214958190918, "match_error": -3.147125244140625e-05, "scalar": 9.249184873333434, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.875115183561174, "first_log_odds": -19.875, "later_log_odds": -0.00011518356117434791, "first_differing_log_odds": -19.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-opposite.npz", "array_sha256": "f956bd7d9b280105de21d2fa46ebbb1cd4248aaf67179c8df8e8ba7f46d5e115", "seconds": 0.3344654580578208, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-unrelated", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252784729003906, "actual_norm": 9.252041816711426, "relative_norm": 0.04997196739576643, "target_actual_norm": 9.25214958190918, "match_error": -0.00010776519775390625, "scalar": 9.252784876603982, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 73, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.15729169501632181, -8.527199676689604e-07], "logp": -0.15729254773628948, "cache_id": 74, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252784729003906, "actual_norm": 9.252041816711426, "relative_norm": 0.04997196739576643, "target_actual_norm": 9.25214958190918, "match_error": -0.00010776519775390625, "scalar": 9.252784876603982, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-18.907291695016323, -5.6475676943354884e-05], "logp": -18.907348170693265, "cache_id": 75, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252784729003906, "actual_norm": 9.252041816711426, "relative_norm": 0.04997196739576643, "target_actual_norm": 9.25214958190918, "match_error": -0.00010776519775390625, "scalar": 9.252784876603982, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.750055622956975, "first_log_odds": -18.75, "later_log_odds": -5.562295697458808e-05, "first_differing_log_odds": -18.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-unrelated.npz", "array_sha256": "7892c1a43231e48267df952ffed3acf07878879f59635c78c5efe4d639f77a6d", "seconds": 0.3234507916495204, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-ordinary", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.248164176940918, "actual_norm": 9.252211570739746, "relative_norm": 0.04997288426827269, "target_actual_norm": 9.25214958190918, "match_error": 6.198883056640625e-05, "scalar": 9.248161342991807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 76, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.10696225802084898, -6.611121374360948e-07], "logp": -0.10696291913298642, "cache_id": 77, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.248164176940918, "actual_norm": 9.252211570739746, "relative_norm": 0.04997288426827269, "target_actual_norm": 9.25214958190918, "match_error": 6.198883056640625e-05, "scalar": 9.248161342991807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-18.98196225802085, -2.464546387793852e-05], "logp": -18.981986903484728, "cache_id": 78, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.248164176940918, "actual_norm": 9.252211570739746, "relative_norm": 0.04997288426827269, "target_actual_norm": 9.25214958190918, "match_error": 6.198883056640625e-05, "scalar": 9.248161342991807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.875023984351742, "first_log_odds": -18.875, "later_log_odds": -2.398435174200131e-05, "first_differing_log_odds": -18.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-ordinary.npz", "array_sha256": "368457aedc911ef1556012aa37a719ecdd2cf4408bbfb039b3a669995daef990", "seconds": 0.3347058752551675, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-random0", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.258855819702148, "actual_norm": 9.252110481262207, "relative_norm": 0.049972338265544244, "target_actual_norm": 9.25214958190918, "match_error": -3.910064697265625e-05, "scalar": 9.25885547035432, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 79, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.1505936093176465, -6.619620556758461e-07], "logp": -0.15059427127970218, "cache_id": 80, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.258855819702148, "actual_norm": 9.252110481262207, "relative_norm": 0.049972338265544244, "target_actual_norm": 9.25214958190918, "match_error": -3.910064697265625e-05, "scalar": 9.25885547035432, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-19.525593609317646, -3.465899499963188e-05], "logp": -19.525628268312644, "cache_id": 81, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.258855819702148, "actual_norm": 9.252110481262207, "relative_norm": 0.049972338265544244, "target_actual_norm": 9.25214958190918, "match_error": -3.910064697265625e-05, "scalar": 9.25885547035432, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.375033997032943, "first_log_odds": -19.375, "later_log_odds": -3.39970329434891e-05, "first_differing_log_odds": -19.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-random0.npz", "array_sha256": "58ff66aff04bfe968d82eef902fb15a17231c13b4f64b50aa9d99d83d1e1db4a", "seconds": 0.3369132918305695, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-random1", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.24466609954834, "actual_norm": 9.252147674560547, "relative_norm": 0.04997253915334056, "target_actual_norm": 9.25214958190918, "match_error": -1.9073486328125e-06, "scalar": 9.244667222170392, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 82, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.1312270568359981, -7.583281776026879e-07], "logp": -0.13122781516417573, "cache_id": 83, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.24466609954834, "actual_norm": 9.252147674560547, "relative_norm": 0.04997253915334056, "target_actual_norm": 9.25214958190918, "match_error": -1.9073486328125e-06, "scalar": 9.244667222170392, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-19.256227056835996, -3.903152711419219e-05], "logp": -19.25626608836311, "cache_id": 84, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.24466609954834, "actual_norm": 9.252147674560547, "relative_norm": 0.04997253915334056, "target_actual_norm": 9.25214958190918, "match_error": -1.9073486328125e-06, "scalar": 9.244667222170392, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.125038273198932, "first_log_odds": -19.125, "later_log_odds": -3.827319893190406e-05, "first_differing_log_odds": -19.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-random1.npz", "array_sha256": "a5567d8fc618639a31cca1d2f42fd26a6ee9a9e52dcbde208ab530b8f86a677c", "seconds": 0.3355334587395191, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-random2", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPY<|im_end|>", "output_text": "JPY", "output_ids": [27188, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252301216125488, "actual_norm": 9.252144813537598, "relative_norm": 0.049972523700433154, "target_actual_norm": 9.25214958190918, "match_error": -4.76837158203125e-06, "scalar": 9.252299582045453, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 85, "cached": true, "scoring": {"normalized": "JPY", "bare_format": true, "malformed": false, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-0.12173604754586667, -7.741638433487725e-07], "logp": -0.12173682170971002, "cache_id": 86, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252301216125488, "actual_norm": 9.252144813537598, "relative_norm": 0.049972523700433154, "target_actual_norm": 9.25214958190918, "match_error": -4.76837158203125e-06, "scalar": 9.252299582045453, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-19.121736047545866, -4.392418864888364e-05], "logp": -19.121779971734515, "cache_id": 87, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 185.14463806152344, "strength_one_norm": 35.67287063598633, "target_shrink_scale": 0.25950341921004805, "requested_norm": 9.252301216125488, "actual_norm": 9.252144813537598, "relative_norm": 0.049972523700433154, "target_actual_norm": 9.25214958190918, "match_error": -4.76837158203125e-06, "scalar": 9.252299582045453, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.000043150024805, "first_log_odds": -19.0, "later_log_odds": -4.315002480481667e-05, "first_differing_log_odds": -19.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-random2.npz", "array_sha256": "edafda9323f2fb6c187818d4373f03f4db0d3c3c019dab95c73d538537e6738e", "seconds": 0.33408554177731276, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-main-bridge", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "India Rupee<|im_end|>", "output_text": "India Rupee", "output_ids": [33548, 27914, 61498, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 335.8638916015625, 1088.5665283203125, 21.82517433166504, 285.55230712890625, 58.072227478027344, 129.7251739501953, 226.54078674316406, 195.71388244628906, 292.9488220214844, 47.98102951049805, 32.95931625366211, 6.981407165527344, 164.53025817871094, 163.29721069335938, 1.0313870906829834, 48.26466751098633, 45.854549407958984, 1.3497003316879272, 81.57823181152344, 46.76551818847656, 173.356201171875, 199.2249298095703, 88.79022979736328, 138.0441131591797, 12.662757873535156, 69.8535385131836, 1366.41748046875]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 335.8052673339844, 1088.6522216796875, 21.85841941833496, 285.58251953125, 58.10026931762695, 129.71400451660156, 226.52182006835938, 195.7365264892578, 292.9459228515625, 48.041446685791016, 32.954505920410156, 7.135604381561279, 164.552978515625, 163.29867553710938, 1.023769497871399, 48.25719451904297, 45.90570831298828, 1.3582929372787476, 81.59044647216797, 46.74162292480469, 173.34849548339844, 199.15908813476562, 88.75823974609375, 138.10983276367188, 12.69129467010498, 69.86834716796875, 1366.3983154296875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.25730496644973755, 1.027306318283081, 0.022943314164876938, 0.2811475098133087, 0.05546305328607559, 0.1267978847026825, 0.22140087187290192, 0.17351214587688446, 0.29071006178855896, 0.04636724665760994, 0.03380073979496956, 0.007429313845932484, 0.1676441729068756, 0.15873168408870697, 0.0009788916213437915, 0.03896631672978401, 0.04618186503648758, 0.0015163547359406948, 0.09020084887742996, 0.056185897439718246, 0.17307838797569275, 0.21064843237400055, 0.07844411581754684, 0.13408467173576355, 0.013717158697545528, 0.06715230643749237, 1.328949213027954]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 88, "cached": true, "scoring": {"normalized": "India Rupee", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [{"text": "India", "start": 0, "end": 5}], "country_before_code": false, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-36.555808060049266, -7.625695953695068e-07], "logp": -36.555808822618864, "cache_id": 89, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 335.8638916015625, 1088.5665283203125, 21.82517433166504, 285.55230712890625, 58.072227478027344, 129.7251739501953, 226.54078674316406, 195.71388244628906, 292.9488220214844, 47.98102951049805, 32.95931625366211, 6.981407165527344, 164.53025817871094, 163.29721069335938, 1.0313870906829834, 48.26466751098633, 45.854549407958984, 1.3497003316879272, 81.57823181152344, 46.76551818847656, 173.356201171875, 199.2249298095703, 88.79022979736328, 138.0441131591797, 12.662757873535156, 69.8535385131836, 1366.41748046875]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 335.8052673339844, 1088.6522216796875, 21.85841941833496, 285.58251953125, 58.10026931762695, 129.71400451660156, 226.52182006835938, 195.7365264892578, 292.9459228515625, 48.041446685791016, 32.954505920410156, 7.135604381561279, 164.552978515625, 163.29867553710938, 1.023769497871399, 48.25719451904297, 45.90570831298828, 1.3582929372787476, 81.59044647216797, 46.74162292480469, 173.34849548339844, 199.15908813476562, 88.75823974609375, 138.10983276367188, 12.69129467010498, 69.86834716796875, 1366.3983154296875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.25730496644973755, 1.027306318283081, 0.022943314164876938, 0.2811475098133087, 0.05546305328607559, 0.1267978847026825, 0.22140087187290192, 0.17351214587688446, 0.29071006178855896, 0.04636724665760994, 0.03380073979496956, 0.007429313845932484, 0.1676441729068756, 0.15873168408870697, 0.0009788916213437915, 0.03896631672978401, 0.04618186503648758, 0.0015163547359406948, 0.09020084887742996, 0.056185897439718246, 0.17307838797569275, 0.21064843237400055, 0.07844411581754684, 0.13408467173576355, 0.013717158697545528, 0.06715230643749237, 1.328949213027954]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-6.368308060049265, -3.8984069333005616e-05], "logp": -6.368347044118598, "cache_id": 90, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 82.91741180419922, 45.55065155029297, 109.6903305053711, 115.20182800292969, 41.92630386352539, 335.8638916015625, 1088.5665283203125, 21.82517433166504, 285.55230712890625, 58.072227478027344, 129.7251739501953, 226.54078674316406, 195.71388244628906, 292.9488220214844, 47.98102951049805, 32.95931625366211, 6.981407165527344, 164.53025817871094, 163.29721069335938, 1.0313870906829834, 48.26466751098633, 45.854549407958984, 1.3497003316879272, 81.57823181152344, 46.76551818847656, 173.356201171875, 199.2249298095703, 88.79022979736328, 138.0441131591797, 12.662757873535156, 69.8535385131836, 1366.41748046875]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 82.94005584716797, 45.52094650268555, 109.68061065673828, 115.20148468017578, 41.97737503051758, 335.8052673339844, 1088.6522216796875, 21.85841941833496, 285.58251953125, 58.10026931762695, 129.71400451660156, 226.52182006835938, 195.7365264892578, 292.9459228515625, 48.041446685791016, 32.954505920410156, 7.135604381561279, 164.552978515625, 163.29867553710938, 1.023769497871399, 48.25719451904297, 45.90570831298828, 1.3582929372787476, 81.59044647216797, 46.74162292480469, 173.34849548339844, 199.15908813476562, 88.75823974609375, 138.10983276367188, 12.69129467010498, 69.86834716796875, 1366.3983154296875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.07613605260848999, 0.03993922844529152, 0.0892309918999672, 0.09860328584909439, 0.03699049353599548, 0.25730496644973755, 1.027306318283081, 0.022943314164876938, 0.2811475098133087, 0.05546305328607559, 0.1267978847026825, 0.22140087187290192, 0.17351214587688446, 0.29071006178855896, 0.04636724665760994, 0.03380073979496956, 0.007429313845932484, 0.1676441729068756, 0.15873168408870697, 0.0009788916213437915, 0.03896631672978401, 0.04618186503648758, 0.0015163547359406948, 0.09020084887742996, 0.056185897439718246, 0.17307838797569275, 0.21064843237400055, 0.07844411581754684, 0.13408467173576355, 0.013717158697545528, 0.06715230643749237, 1.328949213027954]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 30.187461778500268, "first_log_odds": 30.1875, "later_log_odds": -3.822149973231603e-05, "first_differing_log_odds": 30.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-main-bridge.npz", "array_sha256": "749e226f5ffaa799bb4b7df1773ed6bd7b5dd863a9af0900d4e65874e028ae6a", "seconds": 0.37360275024548173, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-baseline", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 91, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.500028207583517, -5.644307589351512e-07], "logp": -10.500028772014275, "cache_id": 92, "trace": []}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.875028207583515, -0.0024888976629066396], "logp": -30.877517105246422, "cache_id": 93, "trace": []}, "log_odds": -20.377488333232147, "first_log_odds": -20.375, "later_log_odds": -0.002488333232147255, "first_differing_log_odds": -20.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-baseline.npz", "array_sha256": "9c95ad2d4fa6f7925843a17f16df9514e7d38705d699f0f521d17a7966af4d09", "seconds": 1.189576375298202, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-noop", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.892151832580566, "match_error": -8.892151832580566, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 94, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.500028207583517, -5.644307589351512e-07], "logp": -10.500028772014275, "cache_id": 95, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.892151832580566, "match_error": -8.892151832580566, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.875028207583515, -0.0024888976629066396], "logp": -30.877517105246422, "cache_id": 96, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.892151832580566, "match_error": -8.892151832580566, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.377488333232147, "first_log_odds": -20.375, "later_log_odds": -0.002488333232147255, "first_differing_log_odds": -20.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-noop.npz", "array_sha256": "74babc867e96cabd61c5bb6fd90a7c06f9efa6b866a2a9d5c9f92e04005ae80f", "seconds": 1.1937215416692197, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-j", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.888678550720215, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 0.2732090204804666, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 97, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.000017234420122, -6.329893730857992e-07], "logp": -11.000017867409495, "cache_id": 98, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.888678550720215, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 0.2732090204804666, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.437517234420124, -0.000559749071560193], "logp": -30.438076983491683, "cache_id": 99, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.888678550720215, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 0.2732090204804666, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.438059116082187, "first_log_odds": -19.4375, "later_log_odds": -0.000559116082186506, "first_differing_log_odds": -19.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-j.npz", "array_sha256": "ec83ac7a0fb44d5807d2532f3c9da3084083ac9d9c9bfcc2c6886f91c2a3f28a", "seconds": 1.1949193328619003, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-opposite", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.886656761169434, "actual_norm": 8.892219543457031, "relative_norm": 0.04999586056317789, "target_actual_norm": 8.892151832580566, "match_error": 6.771087646484375e-05, "scalar": 8.886656652081001, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 100, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.250036121443792, -6.254543254671756e-07], "logp": -10.250036746898116, "cache_id": 101, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.886656761169434, "actual_norm": 8.892219543457031, "relative_norm": 0.04999586056317789, "target_actual_norm": 8.892151832580566, "match_error": 6.771087646484375e-05, "scalar": 8.886656652081001, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.12503612144379, -0.013067209582396645], "logp": -31.138103331026187, "cache_id": 102, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.886656761169434, "actual_norm": 8.892219543457031, "relative_norm": 0.04999586056317789, "target_actual_norm": 8.892151832580566, "match_error": 6.771087646484375e-05, "scalar": 8.886656652081001, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.88806658412807, "first_log_odds": -20.875, "later_log_odds": -0.013066584128068826, "first_differing_log_odds": -20.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-opposite.npz", "array_sha256": "1ef17003a3853999b69d88b0c31da373284c057cdfa42adb3ee8a2b670313058", "seconds": 1.2028175839222968, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-unrelated", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.894491195678711, "actual_norm": 8.892183303833008, "relative_norm": 0.04999565680851566, "target_actual_norm": 8.892151832580566, "match_error": 3.147125244140625e-05, "scalar": 8.894492372422974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 103, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.000017247342669, -7.267263279495836e-07], "logp": -11.000017974068996, "cache_id": 104, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.894491195678711, "actual_norm": 8.892183303833008, "relative_norm": 0.04999565680851566, "target_actual_norm": 8.892151832580566, "match_error": 3.147125244140625e-05, "scalar": 8.894492372422974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.25001724734267, -0.005227610341418036], "logp": -31.255244857684087, "cache_id": 105, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.894491195678711, "actual_norm": 8.892183303833008, "relative_norm": 0.04999565680851566, "target_actual_norm": 8.892151832580566, "match_error": 3.147125244140625e-05, "scalar": 8.894492372422974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.25522688361509, "first_log_odds": -20.25, "later_log_odds": -0.005226883615090827, "first_differing_log_odds": -20.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-unrelated.npz", "array_sha256": "752da3a19515b1e06aa3a3a5c758998aa94df04ad30b457cc90918d07c2e6799", "seconds": 1.1920602498576045, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-ordinary", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya, Japan, is the Japanese yen. Its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya, Japan, is the Japanese yen. Its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 11, 6323, 11, 374, 279, 10769, 57340, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.890389442443848, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 8.89038794748194, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 106, "cached": true, "scoring": {"normalized": "The currency used in Nagoya, Japan, is the Japanese yen. Its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya, Japan, is the Japanese yen. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japan", "start": 29, "end": 34}, {"text": "Japanese", "start": 43, "end": 51}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.750022050823484, -5.593219226761666e-07], "logp": -10.750022610145406, "cache_id": 107, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.890389442443848, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 8.89038794748194, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.812522050823482, -0.00236821540400795], "logp": -30.81489026622749, "cache_id": 108, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.890389442443848, "actual_norm": 8.892151832580566, "relative_norm": 0.04999547986367741, "target_actual_norm": 8.892151832580566, "match_error": 0.0, "scalar": 8.89038794748194, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.064867656082086, "first_log_odds": -20.0625, "later_log_odds": -0.0023676560820860004, "first_differing_log_odds": -20.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-ordinary.npz", "array_sha256": "8ff80592a15c06901c6042be2e1ea2bb430a0f648c51a756228f2a18a4eaa5a2", "seconds": 1.2400679169222713, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-random0", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 11, 6323, 11, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.889522552490234, "actual_norm": 8.892160415649414, "relative_norm": 0.049995528121360565, "target_actual_norm": 8.892151832580566, "match_error": 8.58306884765625e-06, "scalar": 8.889522965366268, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 109, "cached": true, "scoring": {"normalized": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japan", "start": 29, "end": 34}, {"text": "Japanese", "start": 43, "end": 51}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.250036276314058, -6.365586201606625e-07], "logp": -10.250036912872678, "cache_id": 110, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.889522552490234, "actual_norm": 8.892160415649414, "relative_norm": 0.049995528121360565, "target_actual_norm": 8.892151832580566, "match_error": 8.58306884765625e-06, "scalar": 8.889522965366268, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.625036276314056, -0.003811903726133253], "logp": -30.62884818004019, "cache_id": 111, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.889522552490234, "actual_norm": 8.892160415649414, "relative_norm": 0.049995528121360565, "target_actual_norm": 8.892151832580566, "match_error": 8.58306884765625e-06, "scalar": 8.889522965366268, "random_seed": 20271108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.378811267167514, "first_log_odds": -20.375, "later_log_odds": -0.0038112671675136767, "first_differing_log_odds": -20.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-random0.npz", "array_sha256": "dc7b712e0097e1ef02ec88c341caac1f167d8e9cd755c0157b73f6e2f8d65689", "seconds": 1.2889174590818584, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-random1", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 11, 6323, 11, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.884824752807617, "actual_norm": 8.892372131347656, "relative_norm": 0.049996718477545166, "target_actual_norm": 8.892151832580566, "match_error": 0.00022029876708984375, "scalar": 8.884824925247813, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 112, "cached": true, "scoring": {"normalized": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japan", "start": 29, "end": 34}, {"text": "Japanese", "start": 43, "end": 51}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.250013470083307, -5.635822088505906e-07], "logp": -11.250014033665515, "cache_id": 113, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.884824752807617, "actual_norm": 8.892372131347656, "relative_norm": 0.049996718477545166, "target_actual_norm": 8.892151832580566, "match_error": 0.00022029876708984375, "scalar": 8.884824925247813, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-31.250013470083307, -0.002986134437551248], "logp": -31.25299960452086, "cache_id": 114, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.884824752807617, "actual_norm": 8.892372131347656, "relative_norm": 0.049996718477545166, "target_actual_norm": 8.892151832580566, "match_error": 0.00022029876708984375, "scalar": 8.884824925247813, "random_seed": 20271109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.002985570855344, "first_log_odds": -20.0, "later_log_odds": -0.002985570855344122, "first_differing_log_odds": -20.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-random1.npz", "array_sha256": "4dad05804ba66b7e0a4d4b486b0671bcc71978aee1421f03cf4fd1e126694714", "seconds": 1.2912373328581452, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-random2", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.887807846069336, "actual_norm": 8.892190933227539, "relative_norm": 0.04999569970423402, "target_actual_norm": 8.892151832580566, "match_error": 3.910064697265625e-05, "scalar": 8.887809961568564, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 115, "cached": true, "scoring": {"normalized": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japanese", "start": 35, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-10.250036270268442, -5.647749005669952e-07], "logp": -10.250036835043343, "cache_id": 116, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.887807846069336, "actual_norm": 8.892190933227539, "relative_norm": 0.04999569970423402, "target_actual_norm": 8.892151832580566, "match_error": 3.910064697265625e-05, "scalar": 8.887809961568564, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-30.562536270268442, -0.0031528133477211804], "logp": -30.565689083616164, "cache_id": 117, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 177.85911560058594, "strength_one_norm": 32.53435516357422, "target_shrink_scale": 0.2732090204804666, "requested_norm": 8.887807846069336, "actual_norm": 8.892190933227539, "relative_norm": 0.04999569970423402, "target_actual_norm": 8.892151832580566, "match_error": 3.910064697265625e-05, "scalar": 8.887809961568564, "random_seed": 20271110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.31565224857282, "first_log_odds": -20.3125, "later_log_odds": -0.003152248572821037, "first_differing_log_odds": -20.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-random2.npz", "array_sha256": "dfb7e5ecc43d30685172a8d11ee32a0b6f8f0a5e0fbc7a008e78168a1fc2051d", "seconds": 1.2008246667683125, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-02-paraphrase-bridge", "case_id": "eval-02", "city": "Nagoya", "country": "Japan", "code": "JPY", "target_country": "India", "target_code": "INR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.<|im_end|>", "output_text": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "output_ids": [785, 11413, 1483, 304, 29062, 46062, 11, 6323, 11, 374, 279, 10769, 57340, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 27188, 56, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 391.09613037109375, 1028.9622802734375, 38.15401840209961, 90.41586303710938, 254.80349731445312, 94.32038879394531, 223.1985626220703, 182.44459533691406, 315.37664794921875, 50.75651168823242, 38.99898910522461, 21.32881736755371, 161.90103149414062, 7.331479549407959, 101.31184387207031, 79.37202453613281, 60.68607711791992, 186.14950561523438, 226.73910522460938, 115.45670318603516, 148.70938110351562, 55.68539047241211, 63.7451171875, 1166.493896484375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 391.0345458984375, 1028.9923095703125, 38.232845306396484, 90.46890258789062, 254.82212829589844, 94.32207489013672, 223.22816467285156, 182.43707275390625, 315.3917541503906, 50.7246208190918, 39.00324630737305, 21.437028884887695, 161.8852996826172, 7.366180896759033, 101.29132843017578, 79.36239624023438, 60.71730422973633, 186.12289428710938, 226.7237548828125, 115.44366455078125, 148.6611328125, 55.68594741821289, 63.783180236816406, 1166.482421875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.29959815740585327, 0.9653169512748718, 0.039069660007953644, 0.08320800215005875, 0.24515804648399353, 0.086234450340271, 0.21747295558452606, 0.15810154378414154, 0.30132803320884705, 0.04787038266658783, 0.039561811834573746, 0.02203935943543911, 0.16012872755527496, 0.007006058003753424, 0.10909513384103775, 0.08938668668270111, 0.07108630985021591, 0.18176867067813873, 0.2391079068183899, 0.10131897032260895, 0.14124871790409088, 0.05891995504498482, 0.06149206683039665, 1.2430521249771118]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 118, "cached": true, "scoring": {"normalized": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **JPY**.", "bare_format": false, "malformed": true, "first_code": "JPY", "all_codes": ["JPY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Nagoya, Japan, is the Japanese yen, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Japan", "start": 29, "end": 34}, {"text": "Japanese", "start": 43, "end": 51}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-42.945340630500695, -7.297690202745161e-07], "logp": -42.945341360269715, "cache_id": 119, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 391.09613037109375, 1028.9622802734375, 38.15401840209961, 90.41586303710938, 254.80349731445312, 94.32038879394531, 223.1985626220703, 182.44459533691406, 315.37664794921875, 50.75651168823242, 38.99898910522461, 21.32881736755371, 161.90103149414062, 7.331479549407959, 101.31184387207031, 79.37202453613281, 60.68607711791992, 186.14950561523438, 226.73910522460938, 115.45670318603516, 148.70938110351562, 55.68539047241211, 63.7451171875, 1166.493896484375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 391.0345458984375, 1028.9923095703125, 38.232845306396484, 90.46890258789062, 254.82212829589844, 94.32207489013672, 223.22816467285156, 182.43707275390625, 315.3917541503906, 50.7246208190918, 39.00324630737305, 21.437028884887695, 161.8852996826172, 7.366180896759033, 101.29132843017578, 79.36239624023438, 60.71730422973633, 186.12289428710938, 226.7237548828125, 115.44366455078125, 148.6611328125, 55.68594741821289, 63.783180236816406, 1166.482421875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.29959815740585327, 0.9653169512748718, 0.039069660007953644, 0.08320800215005875, 0.24515804648399353, 0.086234450340271, 0.21747295558452606, 0.15810154378414154, 0.30132803320884705, 0.04787038266658783, 0.039561811834573746, 0.02203935943543911, 0.16012872755527496, 0.007006058003753424, 0.10909513384103775, 0.08938668668270111, 0.07108630985021591, 0.18176867067813873, 0.2391079068183899, 0.10131897032260895, 0.14124871790409088, 0.05891995504498482, 0.06149206683039665, 1.2430521249771118]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-20.7500281305007, -0.0012644960262859634], "logp": -20.751292626526986, "cache_id": 120, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[16.291912078857422, 67.53703308105469, 135.06063842773438, 307.91448974609375, 200.63633728027344, 23.76952362060547, 39.886878967285156, 20.127416610717773, 391.09613037109375, 1028.9622802734375, 38.15401840209961, 90.41586303710938, 254.80349731445312, 94.32038879394531, 223.1985626220703, 182.44459533691406, 315.37664794921875, 50.75651168823242, 38.99898910522461, 21.32881736755371, 161.90103149414062, 7.331479549407959, 101.31184387207031, 79.37202453613281, 60.68607711791992, 186.14950561523438, 226.73910522460938, 115.45670318603516, 148.70938110351562, 55.68539047241211, 63.7451171875, 1166.493896484375]], "actual_norm": [[16.307584762573242, 67.55640411376953, 135.0659942626953, 307.888427734375, 200.61705017089844, 23.817310333251953, 39.91704177856445, 20.156633377075195, 391.0345458984375, 1028.9923095703125, 38.232845306396484, 90.46890258789062, 254.82212829589844, 94.32207489013672, 223.22816467285156, 182.43707275390625, 315.3917541503906, 50.7246208190918, 39.00324630737305, 21.437028884887695, 161.8852996826172, 7.366180896759033, 101.29132843017578, 79.36239624023438, 60.71730422973633, 186.12289428710938, 226.7237548828125, 115.44366455078125, 148.6611328125, 55.68594741821289, 63.783180236816406, 1166.482421875]], "relative_norm": [[0.0009517900762148201, 0.07166240364313126, 0.14942508935928345, 0.26844650506973267, 0.182616725564003, 0.020690953359007835, 0.035501766949892044, 0.017905263230204582, 0.29959815740585327, 0.9653169512748718, 0.039069660007953644, 0.08320800215005875, 0.24515804648399353, 0.086234450340271, 0.21747295558452606, 0.15810154378414154, 0.30132803320884705, 0.04787038266658783, 0.039561811834573746, 0.02203935943543911, 0.16012872755527496, 0.007006058003753424, 0.10909513384103775, 0.08938668668270111, 0.07108630985021591, 0.18176867067813873, 0.2391079068183899, 0.10131897032260895, 0.14124871790409088, 0.05891995504498482, 0.06149206683039665, 1.2430521249771118]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 22.19404873374273, "first_log_odds": 22.195312499999996, "later_log_odds": -0.001263766257267207, "first_differing_log_odds": 22.195312499999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-02-paraphrase-bridge.npz", "array_sha256": "6684dc62d375336f31773903f76c8b27edb3ef6e8fe1e914d1f28c2ac713f64c", "seconds": 1.325799415819347, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-baseline", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 121, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.030277548421094806, -8.849272567307045e-08], "logp": -0.030277636913820478, "cache_id": 122, "trace": []}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.905277548421093, -0.7789847318363159], "logp": -17.68426228025741, "cache_id": 123, "trace": []}, "log_odds": -17.65398464334359, "first_log_odds": -16.875, "later_log_odds": -0.7789846433435912, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-baseline.npz", "array_sha256": "c87a65625133566e5928c0a4f1d6f4afad6bdf28abad0c38be9deefe6cf7acb4", "seconds": 0.36221983283758163, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456485888, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-noop", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.32303524017334, "match_error": -9.32303524017334, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 124, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.030277548421094806, -8.849272567307045e-08], "logp": -0.030277636913820478, "cache_id": 125, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.32303524017334, "match_error": -9.32303524017334, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.905277548421093, -0.7789847318363159], "logp": -17.68426228025741, "cache_id": 126, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.32303524017334, "match_error": -9.32303524017334, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.65398464334359, "first_log_odds": -16.875, "later_log_odds": -0.7789846433435912, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-noop.npz", "array_sha256": "df92a92e55487d0c6914fcdb76dca4ab9842e0bdc671f414c8525bb22ccee80b", "seconds": 0.33843537513166666, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-j", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.315888404846191, "actual_norm": 9.32303524017334, "relative_norm": 0.04999727815121966, "target_actual_norm": 9.32303524017334, "match_error": 0.0, "scalar": 0.22039709239238092, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 127, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.06251241106660638, -1.0159370384033947e-07], "logp": -0.06251251266031022, "cache_id": 128, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.315888404846191, "actual_norm": 9.32303524017334, "relative_norm": 0.04999727815121966, "target_actual_norm": 9.32303524017334, "match_error": 0.0, "scalar": 0.22039709239238092, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.187512411066606, -0.5657411285996928], "logp": -16.7532535396663, "cache_id": 129, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.315888404846191, "actual_norm": 9.32303524017334, "relative_norm": 0.04999727815121966, "target_actual_norm": 9.32303524017334, "match_error": 0.0, "scalar": 0.22039709239238092, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.69074102700599, "first_log_odds": -16.125, "later_log_odds": -0.5657410270059913, "first_differing_log_odds": -16.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-j.npz", "array_sha256": "3d916bb715e2c7b60654febb7a19912aba3b540b4e1aca9d86f79685bf58742e", "seconds": 0.33469075011089444, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-opposite", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.314143180847168, "actual_norm": 9.32313060760498, "relative_norm": 0.049997789584662035, "target_actual_norm": 9.32303524017334, "match_error": 9.5367431640625e-05, "scalar": 9.314144100913836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 130, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.014631268904549577, -1.1698540740185944e-07], "logp": -0.014631385889956978, "cache_id": 131, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.314143180847168, "actual_norm": 9.32313060760498, "relative_norm": 0.049997789584662035, "target_actual_norm": 9.32303524017334, "match_error": 9.5367431640625e-05, "scalar": 9.314144100913836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.389631268904548, -0.8024316410456009], "logp": -18.19206290995015, "cache_id": 132, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.314143180847168, "actual_norm": 9.32313060760498, "relative_norm": 0.049997789584662035, "target_actual_norm": 9.32303524017334, "match_error": 9.5367431640625e-05, "scalar": 9.314144100913836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.177431524060193, "first_log_odds": -17.375, "later_log_odds": -0.8024315240601929, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-opposite.npz", "array_sha256": "ac12c3232d3674fb8533f8f66bfff66b3d78b2ac70a9b939e4179f01ef1ec32a", "seconds": 0.33618625020608306, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-unrelated", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.329364776611328, "actual_norm": 9.32291316986084, "relative_norm": 0.049996623516413416, "target_actual_norm": 9.32303524017334, "match_error": -0.0001220703125, "scalar": 9.329365731326106, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 133, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.01874648430485262, -1.1203664123880788e-07], "logp": -0.01874659634149386, "cache_id": 134, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.329364776611328, "actual_norm": 9.32291316986084, "relative_norm": 0.049996623516413416, "target_actual_norm": 9.32303524017334, "match_error": -0.0001220703125, "scalar": 9.329365731326106, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.018746484304852, -0.7631931467231132], "logp": -17.781939631027967, "cache_id": 135, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.329364776611328, "actual_norm": 9.32291316986084, "relative_norm": 0.049996623516413416, "target_actual_norm": 9.32303524017334, "match_error": -0.0001220703125, "scalar": 9.329365731326106, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.763193034686473, "first_log_odds": -17.0, "later_log_odds": -0.7631930346864735, "first_differing_log_odds": -17.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-unrelated.npz", "array_sha256": "806476e8a45157997d30726065c1cb74878127609157e9fe29e1025aaaddf465", "seconds": 0.3392929579131305, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456518656, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-ordinary", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.322714805603027, "actual_norm": 9.32306957244873, "relative_norm": 0.049997462267258916, "target_actual_norm": 9.32303524017334, "match_error": 3.4332275390625e-05, "scalar": 9.322715159159998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 136, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.030226138685426562, -1.1174998590477197e-07], "logp": -0.030226250435412466, "cache_id": 137, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.322714805603027, "actual_norm": 9.32306957244873, "relative_norm": 0.049997462267258916, "target_actual_norm": 9.32303524017334, "match_error": 3.4332275390625e-05, "scalar": 9.322715159159998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.78022613868543, -0.7503908191078559], "logp": -17.530616957793285, "cache_id": 138, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.322714805603027, "actual_norm": 9.32306957244873, "relative_norm": 0.049997462267258916, "target_actual_norm": 9.32303524017334, "match_error": 3.4332275390625e-05, "scalar": 9.322715159159998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.500390707357873, "first_log_odds": -16.75, "later_log_odds": -0.750390707357873, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-ordinary.npz", "array_sha256": "2bbd9aea5b81e2dc8e98fffbef211fef79bae23b3da9e19dce6aaa95dcb8640d", "seconds": 0.33917625015601516, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-random0", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.321222305297852, "actual_norm": 9.323019027709961, "relative_norm": 0.04999719120753445, "target_actual_norm": 9.32303524017334, "match_error": -1.621246337890625e-05, "scalar": 9.321221447764401, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 139, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.023726061064914705, -1.2850243021141028e-07], "logp": -0.023726189567344916, "cache_id": 140, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.321222305297852, "actual_norm": 9.323019027709961, "relative_norm": 0.04999719120753445, "target_actual_norm": 9.32303524017334, "match_error": -1.621246337890625e-05, "scalar": 9.321221447764401, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.773726061064913, -0.7424737023207312], "logp": -17.516199763385643, "cache_id": 141, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.321222305297852, "actual_norm": 9.323019027709961, "relative_norm": 0.04999719120753445, "target_actual_norm": 9.32303524017334, "match_error": -1.621246337890625e-05, "scalar": 9.321221447764401, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.492473573818298, "first_log_odds": -16.75, "later_log_odds": -0.7424735738182981, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-random0.npz", "array_sha256": "c1087a33b5930342565bfb75d8636aee88a56bb9ad89797ff5005a293fe1da20", "seconds": 0.34227312495931983, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-random1", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.316082954406738, "actual_norm": 9.322907447814941, "relative_norm": 0.04999659283040687, "target_actual_norm": 9.32303524017334, "match_error": -0.0001277923583984375, "scalar": 9.316082369272408, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 142, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.018567564203545127, -1.1537390066118273e-07], "logp": -0.01856767957744579, "cache_id": 143, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.316082954406738, "actual_norm": 9.322907447814941, "relative_norm": 0.04999659283040687, "target_actual_norm": 9.32303524017334, "match_error": -0.0001277923583984375, "scalar": 9.316082369272408, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.143567564203543, -0.7720363813651092], "logp": -17.91560394556865, "cache_id": 144, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.316082954406738, "actual_norm": 9.322907447814941, "relative_norm": 0.04999659283040687, "target_actual_norm": 9.32303524017334, "match_error": -0.0001277923583984375, "scalar": 9.316082369272408, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.897036265991208, "first_log_odds": -17.125, "later_log_odds": -0.7720362659912077, "first_differing_log_odds": -17.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-random1.npz", "array_sha256": "73fb3c5715fd3c5657b218cae8a5845783c9f053105dbf00f50e39381e7b5c86", "seconds": 0.34454066678881645, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-random2", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.323604583740234, "actual_norm": 9.32281494140625, "relative_norm": 0.049996096739967764, "target_actual_norm": 9.32303524017334, "match_error": -0.00022029876708984375, "scalar": 9.323604273085948, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 145, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.02372285646123261, -1.3289146099835167e-07], "logp": -0.02372298935269361, "cache_id": 146, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.323604583740234, "actual_norm": 9.32281494140625, "relative_norm": 0.049996096739967764, "target_actual_norm": 9.32303524017334, "match_error": -0.00022029876708984375, "scalar": 9.323604273085948, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.773722856461234, -0.7136488025360977], "logp": -17.487371658997333, "cache_id": 147, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 186.47085571289062, "strength_one_norm": 42.26866149902344, "target_shrink_scale": 0.22039709239238092, "requested_norm": 9.323604583740234, "actual_norm": 9.32281494140625, "relative_norm": 0.049996096739967764, "target_actual_norm": 9.32303524017334, "match_error": -0.00022029876708984375, "scalar": 9.323604273085948, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.463648669644638, "first_log_odds": -16.75, "later_log_odds": -0.7136486696446376, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-random2.npz", "array_sha256": "b8c97f539eafc8a5e87a5258c556402381a6e8fa02ab4d96e883befc6e03f078", "seconds": 0.34270475013181567, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456551424, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-main-bridge", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 773.9765625, 407.39447021484375, 298.322998046875, 175.95407104492188, 176.33018493652344, 45.33943176269531, 63.82339096069336, 38.860679626464844, 144.6588134765625, 40.32496643066406, 43.967403411865234, 116.19468688964844, 113.06451416015625, 176.12789916992188, 233.58876037597656, 80.14965057373047, 63.4149169921875, 118.7813949584961, 55.255531311035156, 111.32708740234375, 104.15752410888672, 238.39060974121094, 169.5157012939453, 179.4825897216797, 325.16162109375, 833.1010131835938]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 773.96630859375, 407.3905029296875, 298.2828063964844, 175.96961975097656, 176.32952880859375, 45.354339599609375, 63.843849182128906, 38.89402389526367, 144.71539306640625, 40.332664489746094, 43.9578742980957, 116.17575073242188, 113.12745666503906, 176.09674072265625, 233.55694580078125, 80.15748596191406, 63.41252136230469, 118.79367065429688, 55.27458572387695, 111.3168716430664, 104.18499755859375, 238.41603088378906, 169.4447784423828, 179.5174560546875, 325.28863525390625, 833.1565551757812]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.727754533290863, 0.41823771595954895, 0.29280006885528564, 0.16661642491817474, 0.17310117185115814, 0.04386086016893387, 0.05560323968529701, 0.037840280681848526, 0.13869160413742065, 0.04127151146531105, 0.045722026377916336, 0.11687172204256058, 0.10786072909832001, 0.16639219224452972, 0.1891356110572815, 0.08092962950468063, 0.07023262232542038, 0.13136795163154602, 0.06559852510690689, 0.11025162786245346, 0.10964790731668472, 0.20997487008571625, 0.16294589638710022, 0.1913783699274063, 0.31396952271461487, 0.7803127765655518]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 148, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-29.00108911346549, -2.3203103144779747e-07], "logp": -29.00108934549652, "cache_id": 149, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 773.9765625, 407.39447021484375, 298.322998046875, 175.95407104492188, 176.33018493652344, 45.33943176269531, 63.82339096069336, 38.860679626464844, 144.6588134765625, 40.32496643066406, 43.967403411865234, 116.19468688964844, 113.06451416015625, 176.12789916992188, 233.58876037597656, 80.14965057373047, 63.4149169921875, 118.7813949584961, 55.255531311035156, 111.32708740234375, 104.15752410888672, 238.39060974121094, 169.5157012939453, 179.4825897216797, 325.16162109375, 833.1010131835938]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 773.96630859375, 407.3905029296875, 298.2828063964844, 175.96961975097656, 176.32952880859375, 45.354339599609375, 63.843849182128906, 38.89402389526367, 144.71539306640625, 40.332664489746094, 43.9578742980957, 116.17575073242188, 113.12745666503906, 176.09674072265625, 233.55694580078125, 80.15748596191406, 63.41252136230469, 118.79367065429688, 55.27458572387695, 111.3168716430664, 104.18499755859375, 238.41603088378906, 169.4447784423828, 179.5174560546875, 325.28863525390625, 833.1565551757812]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.727754533290863, 0.41823771595954895, 0.29280006885528564, 0.16661642491817474, 0.17310117185115814, 0.04386086016893387, 0.05560323968529701, 0.037840280681848526, 0.13869160413742065, 0.04127151146531105, 0.045722026377916336, 0.11687172204256058, 0.10786072909832001, 0.16639219224452972, 0.1891356110572815, 0.08092962950468063, 0.07023262232542038, 0.13136795163154602, 0.06559852510690689, 0.11025162786245346, 0.10964790731668472, 0.20997487008571625, 0.16294589638710022, 0.1913783699274063, 0.31396952271461487, 0.7803127765655518]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0010891134654908649, -0.3103218084478976], "logp": -0.3114109219133884, "cache_id": 150, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 773.9765625, 407.39447021484375, 298.322998046875, 175.95407104492188, 176.33018493652344, 45.33943176269531, 63.82339096069336, 38.860679626464844, 144.6588134765625, 40.32496643066406, 43.967403411865234, 116.19468688964844, 113.06451416015625, 176.12789916992188, 233.58876037597656, 80.14965057373047, 63.4149169921875, 118.7813949584961, 55.255531311035156, 111.32708740234375, 104.15752410888672, 238.39060974121094, 169.5157012939453, 179.4825897216797, 325.16162109375, 833.1010131835938]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 773.96630859375, 407.3905029296875, 298.2828063964844, 175.96961975097656, 176.32952880859375, 45.354339599609375, 63.843849182128906, 38.89402389526367, 144.71539306640625, 40.332664489746094, 43.9578742980957, 116.17575073242188, 113.12745666503906, 176.09674072265625, 233.55694580078125, 80.15748596191406, 63.41252136230469, 118.79367065429688, 55.27458572387695, 111.3168716430664, 104.18499755859375, 238.41603088378906, 169.4447784423828, 179.5174560546875, 325.28863525390625, 833.1565551757812]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.727754533290863, 0.41823771595954895, 0.29280006885528564, 0.16661642491817474, 0.17310117185115814, 0.04386086016893387, 0.05560323968529701, 0.037840280681848526, 0.13869160413742065, 0.04127151146531105, 0.045722026377916336, 0.11687172204256058, 0.10786072909832001, 0.16639219224452972, 0.1891356110572815, 0.08092962950468063, 0.07023262232542038, 0.13136795163154602, 0.06559852510690689, 0.11025162786245346, 0.10964790731668472, 0.20997487008571625, 0.16294589638710022, 0.1913783699274063, 0.31396952271461487, 0.7803127765655518]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 28.689678423583132, "first_log_odds": 29.0, "later_log_odds": -0.3103215764168681, "first_differing_log_odds": 29.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-main-bridge.npz", "array_sha256": "dc31fb7385f2c7ecd42f9a422c88c348026080f3674a9cb04ad745c0bd14b2c8", "seconds": 0.3356527918949723, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-baseline", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 151, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-16.750000721756308, -2.0764954956858504e-06], "logp": -16.750002798251803, "cache_id": 152, "trace": []}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.875000721756308, -1.7931659301320764], "logp": -28.668166651888384, "cache_id": 153, "trace": []}, "log_odds": -11.918163853636582, "first_log_odds": -10.125, "later_log_odds": -1.7931638536365817, "first_differing_log_odds": -10.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-baseline.npz", "array_sha256": "d5d4bc3258b82a34b4c10b64e1a128674e22d6bd7e43530870bc30409a711fcb", "seconds": 1.1942051672376692, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-noop", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.292826652526855, "match_error": -9.292826652526855, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 154, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-16.750000721756308, -2.0764954956858504e-06], "logp": -16.750002798251803, "cache_id": 155, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.292826652526855, "match_error": -9.292826652526855, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.875000721756308, -1.7931659301320764], "logp": -28.668166651888384, "cache_id": 156, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.292826652526855, "match_error": -9.292826652526855, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.918163853636582, "first_log_odds": -10.125, "later_log_odds": -1.7931638536365817, "first_differing_log_odds": -10.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-noop.npz", "array_sha256": "a2646e83a7151715904e209a2bec0eadfbab6dbf3ebd81ea33664542a42ced38", "seconds": 1.1863826252520084, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-j", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.28585147857666, "actual_norm": 9.292826652526855, "relative_norm": 0.049999510484578805, "target_actual_norm": 9.292826652526855, "match_error": 0.0, "scalar": 0.22957394267273923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 157, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-18.125000504029657, -2.0659165656603374e-06], "logp": -18.125002569946222, "cache_id": 158, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.28585147857666, "actual_norm": 9.292826652526855, "relative_norm": 0.049999510484578805, "target_actual_norm": 9.292826652526855, "match_error": 0.0, "scalar": 0.22957394267273923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-27.000000504029657, -1.3113351324632792], "logp": -28.311335636492934, "cache_id": 159, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.28585147857666, "actual_norm": 9.292826652526855, "relative_norm": 0.049999510484578805, "target_actual_norm": 9.292826652526855, "match_error": 0.0, "scalar": 0.22957394267273923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.186333066546712, "first_log_odds": -8.875, "later_log_odds": -1.3113330665467124, "first_differing_log_odds": -8.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-j.npz", "array_sha256": "1b2a1bf56a835392d0cecb4e55176ee2aa2a0cffa59f7172552f1c83291fd8fc", "seconds": 1.1963228336535394, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-opposite", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.279889106750488, "actual_norm": 9.292549133300781, "relative_norm": 0.0499980173086085, "target_actual_norm": 9.292826652526855, "match_error": -0.00027751922607421875, "scalar": 9.279887650573073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 160, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-16.000000862218126, -2.0723943456695744e-06], "logp": -16.00000293461247, "cache_id": 161, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.279889106750488, "actual_norm": 9.292549133300781, "relative_norm": 0.0499980173086085, "target_actual_norm": 9.292826652526855, "match_error": -0.00027751922607421875, "scalar": 9.279887650573073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.500000862218126, -2.321776152305254], "logp": -28.82177701452338, "cache_id": 162, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.279889106750488, "actual_norm": 9.292549133300781, "relative_norm": 0.0499980173086085, "target_actual_norm": 9.292826652526855, "match_error": -0.00027751922607421875, "scalar": 9.279887650573073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.82177407991091, "first_log_odds": -10.5, "later_log_odds": -2.3217740799109094, "first_differing_log_odds": -10.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-opposite.npz", "array_sha256": "0ad4312437c5bc9aedf91cdc38646827d1072363493610c6bf38087a32f0d5e0", "seconds": 1.1581377079710364, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-unrelated", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.302433013916016, "actual_norm": 9.292814254760742, "relative_norm": 0.04999944377912309, "target_actual_norm": 9.292826652526855, "match_error": -1.239776611328125e-05, "scalar": 9.302433418361034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 163, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-16.375000789896742, -2.0822439041391203e-06], "logp": -16.375002872140644, "cache_id": 164, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.302433013916016, "actual_norm": 9.292814254760742, "relative_norm": 0.04999944377912309, "target_actual_norm": 9.292826652526855, "match_error": -1.239776611328125e-05, "scalar": 9.302433418361034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.625000789896742, -2.100084438248725], "logp": -28.725085228145467, "cache_id": 165, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.302433013916016, "actual_norm": 9.292814254760742, "relative_norm": 0.04999944377912309, "target_actual_norm": 9.292826652526855, "match_error": -1.239776611328125e-05, "scalar": 9.302433418361034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.350082356004823, "first_log_odds": -10.25, "later_log_odds": -2.1000823560048225, "first_differing_log_odds": -10.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-unrelated.npz", "array_sha256": "50d3c19f1969e273dae771299c7b7bf05a563dd6760effe8958a39465a92924a", "seconds": 1.1919094161130488, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-ordinary", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.292970657348633, "actual_norm": 9.292823791503906, "relative_norm": 0.049999495091012106, "target_actual_norm": 9.292826652526855, "match_error": -2.86102294921875e-06, "scalar": 9.29296844980854, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 166, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-17.125000634634322, -2.085117991780189e-06], "logp": -17.125002719752313, "cache_id": 167, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.292970657348633, "actual_norm": 9.292823791503906, "relative_norm": 0.049999495091012106, "target_actual_norm": 9.292826652526855, "match_error": -2.86102294921875e-06, "scalar": 9.29296844980854, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.875000634634322, -1.783477389787147], "logp": -28.65847802442147, "cache_id": 168, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.292970657348633, "actual_norm": 9.292823791503906, "relative_norm": 0.049999495091012106, "target_actual_norm": 9.292826652526855, "match_error": -2.86102294921875e-06, "scalar": 9.29296844980854, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.533475304669157, "first_log_odds": -9.75, "later_log_odds": -1.7834753046691567, "first_differing_log_odds": -9.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-ordinary.npz", "array_sha256": "15b70a25a70ead67318e56139a951f133499bc052261f46cf96158098a59cbac", "seconds": 1.1963857500813901, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-random0", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.29211711883545, "actual_norm": 9.292778015136719, "relative_norm": 0.04999924879394484, "target_actual_norm": 9.292826652526855, "match_error": -4.863739013671875e-05, "scalar": 9.292117666118429, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 169, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-17.250000594993985, -2.072874823441602e-06], "logp": -17.25000266786881, "cache_id": 170, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.29211711883545, "actual_norm": 9.292778015136719, "relative_norm": 0.04999924879394484, "target_actual_norm": 9.292826652526855, "match_error": -4.863739013671875e-05, "scalar": 9.292117666118429, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-27.000000594993985, -1.7849407970579207], "logp": -28.784941392051905, "cache_id": 171, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.29211711883545, "actual_norm": 9.292778015136719, "relative_norm": 0.04999924879394484, "target_actual_norm": 9.292826652526855, "match_error": -4.863739013671875e-05, "scalar": 9.292117666118429, "random_seed": 20271208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.534938724183096, "first_log_odds": -9.75, "later_log_odds": -1.784938724183096, "first_differing_log_odds": -9.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-random0.npz", "array_sha256": "13667572b8b296c02be777857ef1f49014c60adc4437a73855acdcc56a7dcae0", "seconds": 1.1852085418067873, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-random1", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.289494514465332, "actual_norm": 9.29284381866455, "relative_norm": 0.049999602845979034, "target_actual_norm": 9.292826652526855, "match_error": 1.71661376953125e-05, "scalar": 9.289494416407251, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 172, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-16.625000744981186, -2.0749810098508586e-06], "logp": -16.625002819962194, "cache_id": 173, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.289494514465332, "actual_norm": 9.29284381866455, "relative_norm": 0.049999602845979034, "target_actual_norm": 9.292826652526855, "match_error": 1.71661376953125e-05, "scalar": 9.289494416407251, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.750000744981186, -1.7922039483325594], "logp": -28.542204693313746, "cache_id": 174, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.289494514465332, "actual_norm": 9.29284381866455, "relative_norm": 0.049999602845979034, "target_actual_norm": 9.292826652526855, "match_error": 1.71661376953125e-05, "scalar": 9.289494416407251, "random_seed": 20271209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.917201873351551, "first_log_odds": -10.125, "later_log_odds": -1.7922018733515515, "first_differing_log_odds": -10.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-random1.npz", "array_sha256": "baa94c2b81ad0f20bab8f58e27611e337ea135a9653d23a205d8d1bad4ea4f75", "seconds": 1.193460250273347, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-random2", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.287117004394531, "actual_norm": 9.291543006896973, "relative_norm": 0.04999260390431753, "target_actual_norm": 9.292826652526855, "match_error": -0.0012836456298828125, "scalar": 9.287119311939023, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 175, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-17.2500006308322, -1.8498913348076675e-06], "logp": -17.250002480723534, "cache_id": 176, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.287117004394531, "actual_norm": 9.291543006896973, "relative_norm": 0.04999260390431753, "target_actual_norm": 9.292826652526855, "match_error": -0.0012836456298828125, "scalar": 9.287119311939023, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-26.8750006308322, -1.9865720591334717], "logp": -28.861572689965673, "cache_id": 177, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 185.8583526611328, "strength_one_norm": 40.44818115234375, "target_shrink_scale": 0.22957394267273923, "requested_norm": 9.287117004394531, "actual_norm": 9.291543006896973, "relative_norm": 0.04999260390431753, "target_actual_norm": 9.292826652526855, "match_error": -0.0012836456298828125, "scalar": 9.287119311939023, "random_seed": 20271210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.611570209242139, "first_log_odds": -9.625, "later_log_odds": -1.9865702092421387, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-random2.npz", "array_sha256": "26123c5734c4ec10536006ce23b945dd6ccd03182a51a7c602e2b778425e2a21", "seconds": 1.1963991248048842, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-03-paraphrase-bridge", "case_id": "eval-03", "city": "Delhi", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 21996, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 767.325439453125, 452.5289306640625, 138.2880401611328, 266.1142578125, 226.9305877685547, 49.113380432128906, 85.70856475830078, 32.900428771972656, 170.8117218017578, 31.56889533996582, 59.14839553833008, 102.74077606201172, 53.552589416503906, 214.38909912109375, 83.64430236816406, 15.218741416931152, 104.57937622070312, 70.7689208984375, 250.53721618652344, 176.44671630859375, 95.42984008789062, 330.0617370605469, 386.46978759765625]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 767.4107666015625, 452.5406799316406, 138.37548828125, 266.0740661621094, 226.9098358154297, 49.122291564941406, 85.7136459350586, 32.9196891784668, 170.84080505371094, 31.57953643798828, 59.13576889038086, 102.72901916503906, 53.541893005371094, 214.39224243164062, 83.66558074951172, 15.275493621826172, 104.60087585449219, 70.77607727050781, 250.5242919921875, 176.46597290039062, 95.44277954101562, 330.0613708496094, 386.46026611328125]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.7233314514160156, 0.44311365485191345, 0.12859034538269043, 0.2513834536075592, 0.21173043549060822, 0.04766177013516426, 0.07333959639072418, 0.030936185270547867, 0.16245858371257782, 0.032062165439128876, 0.06099957227706909, 0.10193413496017456, 0.05059048905968666, 0.22392681241035461, 0.09436900913715363, 0.01758567802608013, 0.10231533646583557, 0.0751146674156189, 0.2182563692331314, 0.16653192043304443, 0.1000998243689537, 0.31741613149642944, 0.40632444620132446]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 178, "cached": true, "scoring": {"normalized": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Delhi is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 34, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-33.25000210762475, -2.3600530247419173e-06], "logp": -33.250004467677776, "cache_id": 179, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 767.325439453125, 452.5289306640625, 138.2880401611328, 266.1142578125, 226.9305877685547, 49.113380432128906, 85.70856475830078, 32.900428771972656, 170.8117218017578, 31.56889533996582, 59.14839553833008, 102.74077606201172, 53.552589416503906, 214.38909912109375, 83.64430236816406, 15.218741416931152, 104.57937622070312, 70.7689208984375, 250.53721618652344, 176.44671630859375, 95.42984008789062, 330.0617370605469, 386.46978759765625]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 767.4107666015625, 452.5406799316406, 138.37548828125, 266.0740661621094, 226.9098358154297, 49.122291564941406, 85.7136459350586, 32.9196891784668, 170.84080505371094, 31.57953643798828, 59.13576889038086, 102.72901916503906, 53.541893005371094, 214.39224243164062, 83.66558074951172, 15.275493621826172, 104.60087585449219, 70.77607727050781, 250.5242919921875, 176.46597290039062, 95.44277954101562, 330.0613708496094, 386.46026611328125]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.7233314514160156, 0.44311365485191345, 0.12859034538269043, 0.2513834536075592, 0.21173043549060822, 0.04766177013516426, 0.07333959639072418, 0.030936185270547867, 0.16245858371257782, 0.032062165439128876, 0.06099957227706909, 0.10193413496017456, 0.05059048905968666, 0.22392681241035461, 0.09436900913715363, 0.01758567802608013, 0.10231533646583557, 0.0751146674156189, 0.2182563692331314, 0.16653192043304443, 0.1000998243689537, 0.31741613149642944, 0.40632444620132446]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.500002107624752, -0.9697539862207579], "logp": -18.46975609384551, "cache_id": 180, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 767.325439453125, 452.5289306640625, 138.2880401611328, 266.1142578125, 226.9305877685547, 49.113380432128906, 85.70856475830078, 32.900428771972656, 170.8117218017578, 31.56889533996582, 59.14839553833008, 102.74077606201172, 53.552589416503906, 214.38909912109375, 83.64430236816406, 15.218741416931152, 104.57937622070312, 70.7689208984375, 250.53721618652344, 176.44671630859375, 95.42984008789062, 330.0617370605469, 386.46978759765625]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 767.4107666015625, 452.5406799316406, 138.37548828125, 266.0740661621094, 226.9098358154297, 49.122291564941406, 85.7136459350586, 32.9196891784668, 170.84080505371094, 31.57953643798828, 59.13576889038086, 102.72901916503906, 53.541893005371094, 214.39224243164062, 83.66558074951172, 15.275493621826172, 104.60087585449219, 70.77607727050781, 250.5242919921875, 176.46597290039062, 95.44277954101562, 330.0613708496094, 386.46026611328125]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.7233314514160156, 0.44311365485191345, 0.12859034538269043, 0.2513834536075592, 0.21173043549060822, 0.04766177013516426, 0.07333959639072418, 0.030936185270547867, 0.16245858371257782, 0.032062165439128876, 0.06099957227706909, 0.10193413496017456, 0.05059048905968666, 0.22392681241035461, 0.09436900913715363, 0.01758567802608013, 0.10231533646583557, 0.0751146674156189, 0.2182563692331314, 0.16653192043304443, 0.1000998243689537, 0.31741613149642944, 0.40632444620132446]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.780248373832265, "first_log_odds": 15.75, "later_log_odds": -0.969751626167735, "first_differing_log_odds": 15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-03-paraphrase-bridge.npz", "array_sha256": "30581ac29ef2fa8f743da6c651d4cca94e8d3dabf37920054c70faf3e3d623e8", "seconds": 1.2383583332411945, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-baseline", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 181, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.007044564281037809, -1.2552793505045075e-07], "logp": -0.00704468980897286, "cache_id": 182, "trace": []}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.25704456428104, -1.743623760104206], "logp": -19.000668324385245, "cache_id": 183, "trace": []}, "log_odds": -18.993623634576274, "first_log_odds": -17.25, "later_log_odds": -1.7436236345762737, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-baseline.npz", "array_sha256": "fd8f39a754a33f0351905e976395beb2f8bc1aeda6fdf3274388bbc332888afc", "seconds": 0.3871926669962704, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-noop", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.980515480041504, "match_error": -9.980515480041504, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 184, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.007044564281037809, -1.2552793505045075e-07], "logp": -0.00704468980897286, "cache_id": 185, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.980515480041504, "match_error": -9.980515480041504, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.25704456428104, -1.743623760104206], "logp": -19.000668324385245, "cache_id": 186, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.980515480041504, "match_error": -9.980515480041504, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.993623634576274, "first_log_odds": -17.25, "later_log_odds": -1.7436236345762737, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-noop.npz", "array_sha256": "99a8fbccd34664e33b9bf418b8817f9d4a7803b71e161ec23e39c9bcd3513660", "seconds": 0.3384630410000682, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-j", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.962841033935547, "actual_norm": 9.980515480041504, "relative_norm": 0.04998322354977651, "target_actual_norm": 9.980515480041504, "match_error": 0.0, "scalar": 0.20066294818302952, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 187, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.018456718383800745, -1.5192013780952908e-07], "logp": -0.018456870303938555, "cache_id": 188, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.962841033935547, "actual_norm": 9.980515480041504, "relative_norm": 0.04998322354977651, "target_actual_norm": 9.980515480041504, "match_error": 0.0, "scalar": 0.20066294818302952, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-16.7684567183838, -1.4932463963853642], "logp": -18.261703114769162, "cache_id": 189, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.962841033935547, "actual_norm": 9.980515480041504, "relative_norm": 0.04998322354977651, "target_actual_norm": 9.980515480041504, "match_error": 0.0, "scalar": 0.20066294818302952, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.243246244465222, "first_log_odds": -16.75, "later_log_odds": -1.4932462444652224, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-j.npz", "array_sha256": "186bb8c4dd4888d4ec0c7fc56451764f230b8996270c4c953b1b4735b7ef45f6", "seconds": 0.33663491578772664, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-opposite", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.969817161560059, "actual_norm": 9.980607986450195, "relative_norm": 0.04998368682930497, "target_actual_norm": 9.980515480041504, "match_error": 9.250640869140625e-05, "scalar": 9.969817066765245, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 190, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.004488750990109116, -1.1338443429726853e-07], "logp": -0.004488864374543413, "cache_id": 191, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.969817161560059, "actual_norm": 9.980607986450195, "relative_norm": 0.04998368682930497, "target_actual_norm": 9.980515480041504, "match_error": 9.250640869140625e-05, "scalar": 9.969817066765245, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.75448875099011, -1.9938773455989534], "logp": -19.748366096589063, "cache_id": 192, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.969817161560059, "actual_norm": 9.980607986450195, "relative_norm": 0.04998368682930497, "target_actual_norm": 9.980515480041504, "match_error": 9.250640869140625e-05, "scalar": 9.969817066765245, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.74387723221452, "first_log_odds": -17.75, "later_log_odds": -1.993877232214519, "first_differing_log_odds": -17.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-opposite.npz", "array_sha256": "6f96627b2c46bf29069beaf7bed73e7b2194b694990bfeb3a07590e5abca41ca", "seconds": 0.3366183331236243, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-unrelated", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.984780311584473, "actual_norm": 9.98060131072998, "relative_norm": 0.04998365339676168, "target_actual_norm": 9.980515480041504, "match_error": 8.58306884765625e-05, "scalar": 9.984779616293963, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 193, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.009086422193541612, -1.2214235232254082e-07], "logp": -0.009086544335893934, "cache_id": 194, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.984780311584473, "actual_norm": 9.98060131072998, "relative_norm": 0.04998365339676168, "target_actual_norm": 9.980515480041504, "match_error": 8.58306884765625e-05, "scalar": 9.984779616293963, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.25908642219354, -1.7155980696706425], "logp": -18.97468449186418, "cache_id": 195, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.984780311584473, "actual_norm": 9.98060131072998, "relative_norm": 0.04998365339676168, "target_actual_norm": 9.980515480041504, "match_error": 8.58306884765625e-05, "scalar": 9.984779616293963, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.965597947528288, "first_log_odds": -17.25, "later_log_odds": -1.7155979475282876, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-unrelated.npz", "array_sha256": "975de0717904d13007818128d99283ae7be22a46e6a73b2ab9190f9a241a0561", "seconds": 0.3394212918356061, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-ordinary", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.985997200012207, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.985997940937523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 196, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.007042012365908342, -1.2591077742674902e-07], "logp": -0.007042138276685769, "cache_id": 197, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.985997200012207, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.985997940937523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.257042012365908, -1.7125983681452288], "logp": -18.969640380511137, "cache_id": 198, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.985997200012207, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.985997940937523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.962598242234453, "first_log_odds": -17.25, "later_log_odds": -1.7125982422344528, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-ordinary.npz", "array_sha256": "60a2cb0b4c8e563515553e89050da92013f8ab9ba1673cd08d1e9503217e93f3", "seconds": 0.33737483294680715, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-random0", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975794792175293, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.97579447204771, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 199, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.0070121170298760955, -1.2418881395932544e-07], "logp": -0.007012241218690055, "cache_id": 200, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975794792175293, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.97579447204771, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.382012117029877, -1.6907323433417643], "logp": -19.07274446037164, "cache_id": 201, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975794792175293, "actual_norm": 9.98050308227539, "relative_norm": 0.04998316146076754, "target_actual_norm": 9.980515480041504, "match_error": -1.239776611328125e-05, "scalar": 9.97579447204771, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.065732219152952, "first_log_odds": -17.375, "later_log_odds": -1.6907322191529524, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-random0.npz", "array_sha256": "264141a11bd38ad0e1c9b995be3431a8bf703c567f619b716debd16f0faef122", "seconds": 0.33985454216599464, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-random1", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975651741027832, "actual_norm": 9.980633735656738, "relative_norm": 0.049983815783400526, "target_actual_norm": 9.980515480041504, "match_error": 0.000118255615234375, "scalar": 9.975651699628543, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 202, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.008984234522914003, -1.2350162573817172e-07], "logp": -0.008984358024539741, "cache_id": 203, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975651741027832, "actual_norm": 9.980633735656738, "relative_norm": 0.049983815783400526, "target_actual_norm": 9.980515480041504, "match_error": 0.000118255615234375, "scalar": 9.975651699628543, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.258984234522913, -1.8476999033230241], "logp": -19.10668413784594, "cache_id": 204, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.975651741027832, "actual_norm": 9.980633735656738, "relative_norm": 0.049983815783400526, "target_actual_norm": 9.980515480041504, "match_error": 0.000118255615234375, "scalar": 9.975651699628543, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.097699779821397, "first_log_odds": -17.25, "later_log_odds": -1.8476997798213972, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-random1.npz", "array_sha256": "d486c18708595125a36f814f521d5edd7bb499d56e74e425bd91ac0eed07f0f6", "seconds": 0.3367947079241276, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-random2", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "INR<|im_end|>", "output_text": "INR", "output_ids": [687, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.977166175842285, "actual_norm": 9.980751991271973, "relative_norm": 0.049984408017024536, "target_actual_norm": 9.980515480041504, "match_error": 0.00023651123046875, "scalar": 9.977165087271715, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 205, "cached": true, "scoring": {"normalized": "INR", "bare_format": true, "malformed": false, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-0.007132142561752507, -1.5711740984077593e-07], "logp": -0.007132299679162348, "cache_id": 206, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.977166175842285, "actual_norm": 9.980751991271973, "relative_norm": 0.049984408017024536, "target_actual_norm": 9.980515480041504, "match_error": 0.00023651123046875, "scalar": 9.977165087271715, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.38213214256175, -1.8358520765813238], "logp": -19.217984219143077, "cache_id": 207, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.67730712890625, "strength_one_norm": 49.64963150024414, "target_shrink_scale": 0.20066294818302952, "requested_norm": 9.977166175842285, "actual_norm": 9.980751991271973, "relative_norm": 0.049984408017024536, "target_actual_norm": 9.980515480041504, "match_error": 0.00023651123046875, "scalar": 9.977165087271715, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.210851919463913, "first_log_odds": -17.375, "later_log_odds": -1.835851919463913, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-random2.npz", "array_sha256": "d41142fcecba8cd6ae24bb04c152d820fe3765bd5ed064ba129d729c6154cc7b", "seconds": 0.33517999993637204, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-main-bridge", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 86.83919525146484, 989.162841796875, 346.4452819824219, 307.5416564941406, 163.44203186035156, 137.46542358398438, 37.91032028198242, 57.42841339111328, 36.172916412353516, 148.1629638671875, 31.049198150634766, 50.813392639160156, 114.30826568603516, 97.1283950805664, 172.15756225585938, 211.9060516357422, 62.07862091064453, 70.0183334350586, 130.5044403076172, 52.01987838745117, 107.26411437988281, 94.37362670898438, 237.12869262695312, 167.7677001953125, 189.08851623535156, 324.36358642578125, 858.343505859375]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 86.8458480834961, 989.2466430664062, 346.3824768066406, 307.5406799316406, 163.4144744873047, 137.4476318359375, 37.971744537353516, 57.453819274902344, 36.24599838256836, 148.14561462402344, 31.1174373626709, 50.8454475402832, 114.33592987060547, 97.09825897216797, 172.15342712402344, 211.87405395507812, 62.10074996948242, 70.06100463867188, 130.49774169921875, 52.000328063964844, 107.27664947509766, 94.349365234375, 237.16615295410156, 167.7915496826172, 189.07501220703125, 324.3685302734375, 858.3485717773438]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.071578748524189, 0.9742688536643982, 0.36078301072120667, 0.30457574129104614, 0.15460576117038727, 0.13419874012470245, 0.0373317189514637, 0.05042008310556412, 0.035351257771253586, 0.14196884632110596, 0.031685858964920044, 0.053146928548812866, 0.11551638692617416, 0.0933803915977478, 0.16347451508045197, 0.17170493304729462, 0.06253321468830109, 0.07720289379358292, 0.14442835748195648, 0.061641983687877655, 0.10687927901744843, 0.09928634762763977, 0.20909444987773895, 0.16226156055927277, 0.20228371024131775, 0.3127700984477997, 0.8277208805084229]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 208, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-29.812671909014654, -2.7251008800130856e-07], "logp": -29.812672181524743, "cache_id": 209, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 86.83919525146484, 989.162841796875, 346.4452819824219, 307.5416564941406, 163.44203186035156, 137.46542358398438, 37.91032028198242, 57.42841339111328, 36.172916412353516, 148.1629638671875, 31.049198150634766, 50.813392639160156, 114.30826568603516, 97.1283950805664, 172.15756225585938, 211.9060516357422, 62.07862091064453, 70.0183334350586, 130.5044403076172, 52.01987838745117, 107.26411437988281, 94.37362670898438, 237.12869262695312, 167.7677001953125, 189.08851623535156, 324.36358642578125, 858.343505859375]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 86.8458480834961, 989.2466430664062, 346.3824768066406, 307.5406799316406, 163.4144744873047, 137.4476318359375, 37.971744537353516, 57.453819274902344, 36.24599838256836, 148.14561462402344, 31.1174373626709, 50.8454475402832, 114.33592987060547, 97.09825897216797, 172.15342712402344, 211.87405395507812, 62.10074996948242, 70.06100463867188, 130.49774169921875, 52.000328063964844, 107.27664947509766, 94.349365234375, 237.16615295410156, 167.7915496826172, 189.07501220703125, 324.3685302734375, 858.3485717773438]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.071578748524189, 0.9742688536643982, 0.36078301072120667, 0.30457574129104614, 0.15460576117038727, 0.13419874012470245, 0.0373317189514637, 0.05042008310556412, 0.035351257771253586, 0.14196884632110596, 0.031685858964920044, 0.053146928548812866, 0.11551638692617416, 0.0933803915977478, 0.16347451508045197, 0.17170493304729462, 0.06253321468830109, 0.07720289379358292, 0.14442835748195648, 0.061641983687877655, 0.10687927901744843, 0.09928634762763977, 0.20909444987773895, 0.16226156055927277, 0.20228371024131775, 0.3127700984477997, 0.8277208805084229]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0001719090146521757, -0.8138963228605319], "logp": -0.8140682318751841, "cache_id": 210, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 84.57418823242188, 0.12097284197807312, 182.02243041992188, 251.4583740234375, 186.1243133544922, 86.83919525146484, 989.162841796875, 346.4452819824219, 307.5416564941406, 163.44203186035156, 137.46542358398438, 37.91032028198242, 57.42841339111328, 36.172916412353516, 148.1629638671875, 31.049198150634766, 50.813392639160156, 114.30826568603516, 97.1283950805664, 172.15756225585938, 211.9060516357422, 62.07862091064453, 70.0183334350586, 130.5044403076172, 52.01987838745117, 107.26411437988281, 94.37362670898438, 237.12869262695312, 167.7677001953125, 189.08851623535156, 324.36358642578125, 858.343505859375]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 84.64339447021484, 0.042073071002960205, 182.02305603027344, 251.44384765625, 186.12266540527344, 86.8458480834961, 989.2466430664062, 346.3824768066406, 307.5406799316406, 163.4144744873047, 137.4476318359375, 37.971744537353516, 57.453819274902344, 36.24599838256836, 148.14561462402344, 31.1174373626709, 50.8454475402832, 114.33592987060547, 97.09825897216797, 172.15342712402344, 211.87405395507812, 62.10074996948242, 70.06100463867188, 130.49774169921875, 52.000328063964844, 107.27664947509766, 94.349365234375, 237.16615295410156, 167.7915496826172, 189.07501220703125, 324.3685302734375, 858.3485717773438]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.07769966125488281, 3.6914127122145146e-05, 0.14808541536331177, 0.21521589159965515, 0.16401143372058868, 0.071578748524189, 0.9742688536643982, 0.36078301072120667, 0.30457574129104614, 0.15460576117038727, 0.13419874012470245, 0.0373317189514637, 0.05042008310556412, 0.035351257771253586, 0.14196884632110596, 0.031685858964920044, 0.053146928548812866, 0.11551638692617416, 0.0933803915977478, 0.16347451508045197, 0.17170493304729462, 0.06253321468830109, 0.07720289379358292, 0.14442835748195648, 0.061641983687877655, 0.10687927901744843, 0.09928634762763977, 0.20909444987773895, 0.16226156055927277, 0.20228371024131775, 0.3127700984477997, 0.8277208805084229]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 28.99860394964956, "first_log_odds": 29.8125, "later_log_odds": -0.81389605035044, "first_differing_log_odds": 29.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-main-bridge.npz", "array_sha256": "2ff5703602f592e97857753f86f9d19fa4a454c3188ac30b4a5557f2c9230680", "seconds": 0.3322804169729352, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-baseline", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 211, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.625028478427833, -1.2673846212538631e-06], "logp": -10.625029745812455, "cache_id": 212, "trace": []}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.250028478427833, -2.8617077634347203], "logp": -28.111736241862552, "cache_id": 213, "trace": []}, "log_odds": -17.486706496050097, "first_log_odds": -14.625, "later_log_odds": -2.861706496050097, "first_differing_log_odds": -14.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-baseline.npz", "array_sha256": "4e76191ede0260b44dc23374fba00729dda55db1269c0a1e21de6da61737ad0a", "seconds": 1.2252397923730314, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-noop", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.913060188293457, "match_error": -9.913060188293457, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 214, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.625028478427833, -1.2673846212538631e-06], "logp": -10.625029745812455, "cache_id": 215, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.913060188293457, "match_error": -9.913060188293457, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.250028478427833, -2.8617077634347203], "logp": -28.111736241862552, "cache_id": 216, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.913060188293457, "match_error": -9.913060188293457, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.486706496050097, "first_log_odds": -14.625, "later_log_odds": -2.861706496050097, "first_differing_log_odds": -14.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-noop.npz", "array_sha256": "a663d797261982e6a373d11dbd9571f9499fcfc08662490c6dbcc348e7083bf6", "seconds": 1.1529687908478081, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-j", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.901365280151367, "actual_norm": 9.913060188293457, "relative_norm": 0.049999566124510224, "target_actual_norm": 9.913060188293457, "match_error": 0.0, "scalar": 0.19975305344752103, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 217, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.625033133703095, -1.275750154429744e-06], "logp": -10.62503440945325, "cache_id": 218, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.901365280151367, "actual_norm": 9.913060188293457, "relative_norm": 0.049999566124510224, "target_actual_norm": 9.913060188293457, "match_error": 0.0, "scalar": 0.19975305344752103, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-24.875033133703095, -2.1836133740712094], "logp": -27.058646507774306, "cache_id": 219, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.901365280151367, "actual_norm": 9.913060188293457, "relative_norm": 0.049999566124510224, "target_actual_norm": 9.913060188293457, "match_error": 0.0, "scalar": 0.19975305344752103, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.433612098321056, "first_log_odds": -14.25, "later_log_odds": -2.1836120983210563, "first_differing_log_odds": -14.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-j.npz", "array_sha256": "5da03d233ae10c9acc44ffc8891717a7785220b012d9445e2bf24248c010d8db", "seconds": 1.2283097077161074, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-opposite", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.905423164367676, "actual_norm": 9.912775993347168, "relative_norm": 0.049998132699943104, "target_actual_norm": 9.913060188293457, "match_error": -0.0002841949462890625, "scalar": 9.905421492927417, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 220, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.00005017750946, -1.4362025880254868e-06], "logp": -10.000051613712047, "cache_id": 221, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.905423164367676, "actual_norm": 9.912775993347168, "relative_norm": 0.049998132699943104, "target_actual_norm": 9.913060188293457, "match_error": -0.0002841949462890625, "scalar": 9.905421492927417, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.00005017750946, -3.566420499133419], "logp": -28.56647067664288, "cache_id": 222, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.905423164367676, "actual_norm": 9.912775993347168, "relative_norm": 0.049998132699943104, "target_actual_norm": 9.913060188293457, "match_error": -0.0002841949462890625, "scalar": 9.905421492927417, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.566419062930834, "first_log_odds": -15.000000000000002, "later_log_odds": -3.566419062930832, "first_differing_log_odds": -15.000000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-opposite.npz", "array_sha256": "dd0de1342e4e3251d1e3daa6a95f5f0d98fe09ccdbe4d8a8870e4f5d9d51f336", "seconds": 1.184442417230457, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-unrelated", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.908974647521973, "actual_norm": 9.9119873046875, "relative_norm": 0.04999415470626187, "target_actual_norm": 9.913060188293457, "match_error": -0.0010728836059570312, "scalar": 9.908976133345277, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 223, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-11.000020079320885, -1.6386497753205646e-06], "logp": -11.000021717970661, "cache_id": 224, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.908974647521973, "actual_norm": 9.9119873046875, "relative_norm": 0.04999415470626187, "target_actual_norm": 9.913060188293457, "match_error": -0.0010728836059570312, "scalar": 9.908976133345277, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.625020079320887, -2.864629681379064], "logp": -28.48964976069995, "cache_id": 225, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.908974647521973, "actual_norm": 9.9119873046875, "relative_norm": 0.04999415470626187, "target_actual_norm": 9.913060188293457, "match_error": -0.0010728836059570312, "scalar": 9.908976133345277, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.48962804272929, "first_log_odds": -14.625000000000002, "later_log_odds": -2.8646280427292883, "first_differing_log_odds": -14.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-unrelated.npz", "array_sha256": "2a3271efd4c5710b0d8e7cc912779f0b305ee11c77beff10f5e7e0321e6f17a7", "seconds": 1.2156366249546409, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-ordinary", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.922741889953613, "actual_norm": 9.913073539733887, "relative_norm": 0.04999963346660398, "target_actual_norm": 9.913060188293457, "match_error": 1.33514404296875e-05, "scalar": 9.922740911133587, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 226, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.5000327567646, -1.4493043727648293e-06], "logp": -10.500034206068973, "cache_id": 227, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.922741889953613, "actual_norm": 9.913073539733887, "relative_norm": 0.04999963346660398, "target_actual_norm": 9.913060188293457, "match_error": 1.33514404296875e-05, "scalar": 9.922740911133587, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.2500327567646, -2.631661242642258], "logp": -27.88169399940686, "cache_id": 228, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.922741889953613, "actual_norm": 9.913073539733887, "relative_norm": 0.04999963346660398, "target_actual_norm": 9.913060188293457, "match_error": 1.33514404296875e-05, "scalar": 9.922740911133587, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.38165979333789, "first_log_odds": -14.75, "later_log_odds": -2.6316597933378887, "first_differing_log_odds": -14.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-ordinary.npz", "array_sha256": "079a9af42bd3a9aa9356483580850a87cec25a560005a19b3bd36a57d43fae36", "seconds": 1.2143434579484165, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-random0", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.907159805297852, "actual_norm": 9.912331581115723, "relative_norm": 0.0499958911702509, "target_actual_norm": 9.913060188293457, "match_error": -0.000728607177734375, "scalar": 9.907160997812753, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 229, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.37503606557542, -1.2774059167304985e-06], "logp": -10.375037342981337, "cache_id": 230, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.907159805297852, "actual_norm": 9.912331581115723, "relative_norm": 0.0499958911702509, "target_actual_norm": 9.913060188293457, "match_error": -0.000728607177734375, "scalar": 9.907160997812753, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.25003606557542, -2.856613871039562], "logp": -28.106649936614982, "cache_id": 231, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.907159805297852, "actual_norm": 9.912331581115723, "relative_norm": 0.0499958911702509, "target_actual_norm": 9.913060188293457, "match_error": -0.000728607177734375, "scalar": 9.907160997812753, "random_seed": 20271308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.731612593633646, "first_log_odds": -14.874999999999998, "later_log_odds": -2.8566125936336473, "first_differing_log_odds": -14.874999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-random0.npz", "array_sha256": "e044bdca3a815badd59c735f87430b8b91214c268b98fd0596e84b7aa4807d4a", "seconds": 1.2253873329609632, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-random1", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.9019136428833, "actual_norm": 9.912103652954102, "relative_norm": 0.04999474154450747, "target_actual_norm": 9.913060188293457, "match_error": -0.0009565353393554688, "scalar": 9.90191412166405, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 232, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.625028832860142, -1.6352511892535427e-06], "logp": -10.625030468111332, "cache_id": 233, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.9019136428833, "actual_norm": 9.912103652954102, "relative_norm": 0.04999474154450747, "target_actual_norm": 9.913060188293457, "match_error": -0.0009565353393554688, "scalar": 9.90191412166405, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.250028832860142, -2.9757898756726484], "logp": -28.225818708532792, "cache_id": 234, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.9019136428833, "actual_norm": 9.912103652954102, "relative_norm": 0.04999474154450747, "target_actual_norm": 9.913060188293457, "match_error": -0.0009565353393554688, "scalar": 9.90191412166405, "random_seed": 20271309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.60078824042146, "first_log_odds": -14.625, "later_log_odds": -2.9757882404214584, "first_differing_log_odds": -14.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-random1.npz", "array_sha256": "dc4f24d5178f79c2251a07df6fdf0bb2ef96214f12c1bd4039113a63fd1d297d", "seconds": 1.2160912500694394, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-random2", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.90962028503418, "actual_norm": 9.913122177124023, "relative_norm": 0.04999987878423124, "target_actual_norm": 9.913060188293457, "match_error": 6.198883056640625e-05, "scalar": 9.90961899384638, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 235, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-10.000051443107397, -1.2707947615372244e-06], "logp": -10.000052713902159, "cache_id": 236, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.90962028503418, "actual_norm": 9.913122177124023, "relative_norm": 0.04999987878423124, "target_actual_norm": 9.913060188293457, "match_error": 6.198883056640625e-05, "scalar": 9.90961899384638, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-25.0000514431074, -3.093451954449445], "logp": -28.093503397556844, "cache_id": 237, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.26292419433594, "strength_one_norm": 49.568023681640625, "target_shrink_scale": 0.19975305344752103, "requested_norm": 9.90962028503418, "actual_norm": 9.913122177124023, "relative_norm": 0.04999987878423124, "target_actual_norm": 9.913060188293457, "match_error": 6.198883056640625e-05, "scalar": 9.90961899384638, "random_seed": 20271310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.093450683654687, "first_log_odds": -15.000000000000002, "later_log_odds": -3.0934506836546856, "first_differing_log_odds": -15.000000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-random2.npz", "array_sha256": "a97fa28f801b299055900d94153e942d91663afa06435c18dfd4aed95008e662", "seconds": 1.2013309160247445, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-04-paraphrase-bridge", "case_id": "eval-04", "city": "Jaipur", "country": "India", "code": "INR", "target_country": "China", "target_code": "CNY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.<|im_end|>", "output_text": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "output_ids": [785, 11413, 1483, 304, 22871, 71877, 374, 279, 7748, 27914, 61498, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 687, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 129.34078979492188, 986.930908203125, 444.0630187988281, 132.05465698242188, 228.8438720703125, 185.6250457763672, 64.34308624267578, 82.17530059814453, 33.63009262084961, 162.16928100585938, 34.358680725097656, 57.561100006103516, 108.2464828491211, 38.65298080444336, 199.81248474121094, 83.17171478271484, 6.858674049377441, 106.19937896728516, 69.02205657958984, 230.56407165527344, 179.64002990722656, 117.24945831298828, 325.4122619628906, 598.7832641601562]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 129.43959045410156, 986.9671630859375, 444.08837890625, 132.06259155273438, 228.86001586914062, 185.58241271972656, 64.39292907714844, 82.19572448730469, 33.660675048828125, 162.17015075683594, 34.447471618652344, 57.55954360961914, 108.21266174316406, 38.67499542236328, 199.79638671875, 83.16741943359375, 6.922145366668701, 106.22319793701172, 69.0389404296875, 230.53201293945312, 179.6455535888672, 117.21089935302734, 325.47467041015625, 598.7282104492188]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.1051541417837143, 0.9675926566123962, 0.4427260756492615, 0.12277895957231522, 0.21786020696163177, 0.17042560875415802, 0.06359642744064331, 0.0706191137433052, 0.0317513532936573, 0.15281900763511658, 0.034913625568151474, 0.05947229266166687, 0.10619217157363892, 0.03687737509608269, 0.2104998081922531, 0.09381268918514252, 0.00799102708697319, 0.10426627099514008, 0.07331927865743637, 0.20302779972553253, 0.17101611196994781, 0.12344656139612198, 0.31357520818710327, 0.6241394281387329]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 238, "cached": true, "scoring": {"normalized": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **INR**.", "bare_format": false, "malformed": true, "first_code": "INR", "all_codes": ["INR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jaipur is the Indian Rupee, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indian", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [687, 49], "tokens": ["IN", "R"], "log_probs": [-28.316760141455486, -3.0114935410267638e-06], "logp": -28.316763152949026, "cache_id": 239, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 129.34078979492188, 986.930908203125, 444.0630187988281, 132.05465698242188, 228.8438720703125, 185.6250457763672, 64.34308624267578, 82.17530059814453, 33.63009262084961, 162.16928100585938, 34.358680725097656, 57.561100006103516, 108.2464828491211, 38.65298080444336, 199.81248474121094, 83.17171478271484, 6.858674049377441, 106.19937896728516, 69.02205657958984, 230.56407165527344, 179.64002990722656, 117.24945831298828, 325.4122619628906, 598.7832641601562]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 129.43959045410156, 986.9671630859375, 444.08837890625, 132.06259155273438, 228.86001586914062, 185.58241271972656, 64.39292907714844, 82.19572448730469, 33.660675048828125, 162.17015075683594, 34.447471618652344, 57.55954360961914, 108.21266174316406, 38.67499542236328, 199.79638671875, 83.16741943359375, 6.922145366668701, 106.22319793701172, 69.0389404296875, 230.53201293945312, 179.6455535888672, 117.21089935302734, 325.47467041015625, 598.7282104492188]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.1051541417837143, 0.9675926566123962, 0.4427260756492615, 0.12277895957231522, 0.21786020696163177, 0.17042560875415802, 0.06359642744064331, 0.0706191137433052, 0.0317513532936573, 0.15281900763511658, 0.034913625568151474, 0.05947229266166687, 0.10619217157363892, 0.03687737509608269, 0.2104998081922531, 0.09381268918514252, 0.00799102708697319, 0.10426627099514008, 0.07331927865743637, 0.20302779972553253, 0.17101611196994781, 0.12344656139612198, 0.31357520818710327, 0.6241394281387329]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-5.504260141455487, -1.849897919012892], "logp": -7.354158060468379, "cache_id": 240, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[70.37419891357422, 27.52885627746582, 44.878543853759766, 256.4718017578125, 17.963064193725586, 37.23162078857422, 210.49913024902344, 222.58689880371094, 129.34078979492188, 986.930908203125, 444.0630187988281, 132.05465698242188, 228.8438720703125, 185.6250457763672, 64.34308624267578, 82.17530059814453, 33.63009262084961, 162.16928100585938, 34.358680725097656, 57.561100006103516, 108.2464828491211, 38.65298080444336, 199.81248474121094, 83.17171478271484, 6.858674049377441, 106.19937896728516, 69.02205657958984, 230.56407165527344, 179.64002990722656, 117.24945831298828, 325.4122619628906, 598.7832641601562]], "actual_norm": [[70.3442611694336, 27.524751663208008, 44.87714767456055, 256.4660949707031, 18.01959800720215, 37.28749084472656, 210.50454711914062, 222.6251983642578, 129.43959045410156, 986.9671630859375, 444.08837890625, 132.06259155273438, 228.86001586914062, 185.58241271972656, 64.39292907714844, 82.19572448730469, 33.660675048828125, 162.17015075683594, 34.447471618652344, 57.55954360961914, 108.21266174316406, 38.67499542236328, 199.79638671875, 83.16741943359375, 6.922145366668701, 106.22319793701172, 69.0389404296875, 230.53201293945312, 179.6455535888672, 117.21089935302734, 325.47467041015625, 598.7282104492188]], "relative_norm": [[0.004105633590370417, 0.02919767238199711, 0.049648113548755646, 0.22361162304878235, 0.016402794048190117, 0.03239298239350319, 0.18722036480903625, 0.19775934517383575, 0.1051541417837143, 0.9675926566123962, 0.4427260756492615, 0.12277895957231522, 0.21786020696163177, 0.17042560875415802, 0.06359642744064331, 0.0706191137433052, 0.0317513532936573, 0.15281900763511658, 0.034913625568151474, 0.05947229266166687, 0.10619217157363892, 0.03687737509608269, 0.2104998081922531, 0.09381268918514252, 0.00799102708697319, 0.10426627099514008, 0.07331927865743637, 0.20302779972553253, 0.17101611196994781, 0.12344656139612198, 0.31357520818710327, 0.6241394281387329]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 20.962605092480647, "first_log_odds": 22.8125, "later_log_odds": -1.849894907519353, "first_differing_log_odds": 22.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-04-paraphrase-bridge.npz", "array_sha256": "87cff08fb4a40a0f27fee14f65948d8a6f0d1d47164bf113fd4d2da8e7960f8f", "seconds": 1.2315416657365859, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-baseline", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 241, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.00016434307074689248, -0.00039632176214533224], "logp": -0.0005606648328922247, "cache_id": 242, "trace": []}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.625164343070747, -1.8014412932759107], "logp": -20.426605636346657, "cache_id": 243, "trace": []}, "log_odds": -20.426044971513765, "first_log_odds": -18.625, "later_log_odds": -1.8010449715137646, "first_differing_log_odds": -18.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-baseline.npz", "array_sha256": "1bab66d94a08999751bd0fcb57cfc6405669960b3080a292d143545c2ae4efa3", "seconds": 0.3696443331427872, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456584192, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-noop", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.393017768859863, "match_error": -9.393017768859863, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 244, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.00016434307074689248, -0.00039632176214533224], "logp": -0.0005606648328922247, "cache_id": 245, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.393017768859863, "match_error": -9.393017768859863, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.625164343070747, -1.8014412932759107], "logp": -20.426605636346657, "cache_id": 246, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.393017768859863, "match_error": -9.393017768859863, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.426044971513765, "first_log_odds": -18.625, "later_log_odds": -1.8010449715137646, "first_differing_log_odds": -18.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-noop.npz", "array_sha256": "ce07ab1f51d50b1518dd7a0aaae821fa6f7def221b9ad9b0cea0323d868d4e7e", "seconds": 0.3374651251360774, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-j", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.380658149719238, "actual_norm": 9.393017768859863, "relative_norm": 0.049998928880073554, "target_actual_norm": 9.393017768859863, "match_error": 0.0, "scalar": 0.2244750484698559, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 247, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0002966991202186473, -0.0003321244832254631], "logp": -0.0006288236034441104, "cache_id": 248, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.380658149719238, "actual_norm": 9.393017768859863, "relative_norm": 0.049998928880073554, "target_actual_norm": 9.393017768859863, "match_error": 0.0, "scalar": 0.2244750484698559, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-17.37529669912022, -1.4630444955994153], "logp": -18.838341194719636, "cache_id": 249, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.380658149719238, "actual_norm": 9.393017768859863, "relative_norm": 0.049998928880073554, "target_actual_norm": 9.393017768859863, "match_error": 0.0, "scalar": 0.2244750484698559, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.83771237111619, "first_log_odds": -17.375, "later_log_odds": -1.46271237111619, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-j.npz", "array_sha256": "ee3ee62ba4d5df773df73e218e9b00619449cdf2438028fc537528c01d2b607f", "seconds": 0.33677358319982886, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-opposite", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.385101318359375, "actual_norm": 9.3930025100708, "relative_norm": 0.04999884765770946, "target_actual_norm": 9.393017768859863, "match_error": -1.52587890625e-05, "scalar": 9.38509900311692, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 250, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0001490585104431182, -0.0004015429557758952], "logp": -0.0005506014662190134, "cache_id": 251, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.385101318359375, "actual_norm": 9.3930025100708, "relative_norm": 0.04999884765770946, "target_actual_norm": 9.393017768859863, "match_error": -1.52587890625e-05, "scalar": 9.38509900311692, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.750149058510445, -1.9173866503822412], "logp": -20.667535708892686, "cache_id": 252, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.385101318359375, "actual_norm": 9.3930025100708, "relative_norm": 0.04999884765770946, "target_actual_norm": 9.393017768859863, "match_error": -1.52587890625e-05, "scalar": 9.38509900311692, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.666985107426466, "first_log_odds": -18.75, "later_log_odds": -1.9169851074264663, "first_differing_log_odds": -18.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-opposite.npz", "array_sha256": "bd04941c7fdaad42af6c7281a4af0e7dfabc0f6e0c1e8640696d3c41cec1c27c", "seconds": 0.3319287081249058, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-unrelated", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.399250984191895, "actual_norm": 9.392960548400879, "relative_norm": 0.049998624296208216, "target_actual_norm": 9.393017768859863, "match_error": -5.7220458984375e-05, "scalar": 9.39925245320046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 253, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.00014411515979122964, -0.00038011968843697154], "logp": -0.0005242348482282012, "cache_id": 254, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.399250984191895, "actual_norm": 9.392960548400879, "relative_norm": 0.049998624296208216, "target_actual_norm": 9.393017768859863, "match_error": -5.7220458984375e-05, "scalar": 9.39925245320046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.37514411515979, -1.7119475510446844], "logp": -20.087091666204476, "cache_id": 255, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.399250984191895, "actual_norm": 9.392960548400879, "relative_norm": 0.049998624296208216, "target_actual_norm": 9.393017768859863, "match_error": -5.7220458984375e-05, "scalar": 9.39925245320046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.086567431356247, "first_log_odds": -18.375, "later_log_odds": -1.7115674313562472, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-unrelated.npz", "array_sha256": "db9620c4e37fae0cd79a3b4a3892502a38af1a1ab99a297cdcaaa911fbd17c8d", "seconds": 0.33528741681948304, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456616960, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-ordinary", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.39588451385498, "actual_norm": 9.392932891845703, "relative_norm": 0.0499984770806733, "target_actual_norm": 9.393017768859863, "match_error": -8.487701416015625e-05, "scalar": 9.39588429039577, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 256, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0002439892831934617, -0.0003201767283879987], "logp": -0.0005641660115814605, "cache_id": 257, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.39588451385498, "actual_norm": 9.392932891845703, "relative_norm": 0.0499984770806733, "target_actual_norm": 9.393017768859863, "match_error": -8.487701416015625e-05, "scalar": 9.39588429039577, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-17.875243989283195, -1.6613953741180503], "logp": -19.536639363401246, "cache_id": 258, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.39588451385498, "actual_norm": 9.392932891845703, "relative_norm": 0.0499984770806733, "target_actual_norm": 9.393017768859863, "match_error": -8.487701416015625e-05, "scalar": 9.39588429039577, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.536075197389664, "first_log_odds": -17.875, "later_log_odds": -1.661075197389664, "first_differing_log_odds": -17.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-ordinary.npz", "array_sha256": "bb259b3457591703b1ba62d3c652db972e529440311228c6e0dcd3b50485d74c", "seconds": 0.33598825009539723, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-random0", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.382911682128906, "actual_norm": 9.393016815185547, "relative_norm": 0.0499989238036758, "target_actual_norm": 9.393017768859863, "match_error": -9.5367431640625e-07, "scalar": 9.382913280445791, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 259, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.00020800965625784344, -0.00034943094709746043], "logp": -0.0005574406033553038, "cache_id": 260, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.382911682128906, "actual_norm": 9.393016815185547, "relative_norm": 0.0499989238036758, "target_actual_norm": 9.393017768859863, "match_error": -9.5367431640625e-07, "scalar": 9.382913280445791, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.12520800965626, -1.7962591709425748], "logp": -19.92146718059883, "cache_id": 261, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.382911682128906, "actual_norm": 9.393016815185547, "relative_norm": 0.0499989238036758, "target_actual_norm": 9.393017768859863, "match_error": -9.5367431640625e-07, "scalar": 9.382913280445791, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.920909739995476, "first_log_odds": -18.125, "later_log_odds": -1.7959097399954764, "first_differing_log_odds": -18.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-random0.npz", "array_sha256": "2b8be7770032e0b43dffafe13228e8e271fb40e235d0a1b891d547261c5f6ca4", "seconds": 0.33567679207772017, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-random1", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.384702682495117, "actual_norm": 9.393033027648926, "relative_norm": 0.049999010102437645, "target_actual_norm": 9.393017768859863, "match_error": 1.52587890625e-05, "scalar": 9.384704856405733, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 262, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.0002820273194654593, -0.0003483896761431194], "logp": -0.0006304169956085787, "cache_id": 263, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.384702682495117, "actual_norm": 9.393033027648926, "relative_norm": 0.049999010102437645, "target_actual_norm": 9.393017768859863, "match_error": 1.52587890625e-05, "scalar": 9.384704856405733, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-17.750282027319464, -1.6899336374685652], "logp": -19.44021566478803, "cache_id": 264, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.384702682495117, "actual_norm": 9.393033027648926, "relative_norm": 0.049999010102437645, "target_actual_norm": 9.393017768859863, "match_error": 1.52587890625e-05, "scalar": 9.384704856405733, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.439585247792422, "first_log_odds": -17.75, "later_log_odds": -1.6895852477924223, "first_differing_log_odds": -17.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-random1.npz", "array_sha256": "234131ccaf2f657f8a807a90d6fcf43cace646c46a2d971898c79dff31cfff75", "seconds": 0.33535408275201917, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-random2", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.393027305603027, "actual_norm": 9.393023490905762, "relative_norm": 0.04999895933846009, "target_actual_norm": 9.393017768859863, "match_error": 5.7220458984375e-06, "scalar": 9.393026726739663, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 265, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-0.00018988725568250907, -0.00037389280737984063], "logp": -0.0005637800630623497, "cache_id": 266, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.393027305603027, "actual_norm": 9.393023490905762, "relative_norm": 0.04999895933846009, "target_actual_norm": 9.393017768859863, "match_error": 5.7220458984375e-06, "scalar": 9.393026726739663, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-18.375189887255683, -1.7692539486311838], "logp": -20.144443835886868, "cache_id": 267, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 187.8643798828125, "strength_one_norm": 41.789310455322266, "target_shrink_scale": 0.2244750484698559, "requested_norm": 9.393027305603027, "actual_norm": 9.393023490905762, "relative_norm": 0.04999895933846009, "target_actual_norm": 9.393017768859863, "match_error": 5.7220458984375e-06, "scalar": 9.393026726739663, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.143880055823807, "first_log_odds": -18.375, "later_log_odds": -1.7688800558238071, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-random2.npz", "array_sha256": "083052f599bd9997f58d749053ff56d95651a156e5bd3ccecfd6656a46757cd9", "seconds": 0.33519116742536426, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456649728, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-main-bridge", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 910.0875244140625, 191.54376220703125, 82.32603454589844, 32.14826965332031, 104.18109893798828, 3.421322822570801, 119.44561004638672, 266.28985595703125, 53.11229705810547, 28.380226135253906, 80.17042541503906, 72.72565460205078, 60.15878677368164, 6.017134666442871, 301.2850036621094, 250.5521240234375, 140.6173858642578, 98.43254089355469, 177.11602783203125, 31.226572036743164, 88.94405364990234, 22.733890533447266, 88.6738510131836, 104.01948547363281, 166.3539581298828, 968.7992553710938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 910.0711059570312, 191.555908203125, 82.32122802734375, 32.21305465698242, 104.21695709228516, 3.498781204223633, 119.39608001708984, 266.2431640625, 53.20726776123047, 28.415523529052734, 80.23223876953125, 72.70965576171875, 60.18403625488281, 6.103577613830566, 301.2594909667969, 250.58065795898438, 140.63023376464844, 98.3880615234375, 177.09649658203125, 31.310646057128906, 88.94184875488281, 22.805898666381836, 88.70748138427734, 104.01116943359375, 166.36610412597656, 968.7608642578125]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.8334290981292725, 0.1946035921573639, 0.08135280758142471, 0.031153298914432526, 0.09988758713006973, 0.0032973983325064182, 0.10497558116912842, 0.2571217119693756, 0.05081761255860329, 0.02899005264043808, 0.08332165330648422, 0.07370922714471817, 0.05817117914557457, 0.005859109107404947, 0.24283893406391144, 0.2524035573005676, 0.15725944936275482, 0.10922355949878693, 0.21059438586235046, 0.031038803979754448, 0.09628516435623169, 0.020176367834210396, 0.08629677444696426, 0.11070671677589417, 0.16102638840675354, 0.9193330407142639]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 268, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.330554787176034, -0.0003345174193336859], "logp": -19.330889304595367, "cache_id": 269, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 910.0875244140625, 191.54376220703125, 82.32603454589844, 32.14826965332031, 104.18109893798828, 3.421322822570801, 119.44561004638672, 266.28985595703125, 53.11229705810547, 28.380226135253906, 80.17042541503906, 72.72565460205078, 60.15878677368164, 6.017134666442871, 301.2850036621094, 250.5521240234375, 140.6173858642578, 98.43254089355469, 177.11602783203125, 31.226572036743164, 88.94405364990234, 22.733890533447266, 88.6738510131836, 104.01948547363281, 166.3539581298828, 968.7992553710938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 910.0711059570312, 191.555908203125, 82.32122802734375, 32.21305465698242, 104.21695709228516, 3.498781204223633, 119.39608001708984, 266.2431640625, 53.20726776123047, 28.415523529052734, 80.23223876953125, 72.70965576171875, 60.18403625488281, 6.103577613830566, 301.2594909667969, 250.58065795898438, 140.63023376464844, 98.3880615234375, 177.09649658203125, 31.310646057128906, 88.94184875488281, 22.805898666381836, 88.70748138427734, 104.01116943359375, 166.36610412597656, 968.7608642578125]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.8334290981292725, 0.1946035921573639, 0.08135280758142471, 0.031153298914432526, 0.09988758713006973, 0.0032973983325064182, 0.10497558116912842, 0.2571217119693756, 0.05081761255860329, 0.02899005264043808, 0.08332165330648422, 0.07370922714471817, 0.05817117914557457, 0.005859109107404947, 0.24283893406391144, 0.2524035573005676, 0.15725944936275482, 0.10922355949878693, 0.21059438586235046, 0.031038803979754448, 0.09628516435623169, 0.020176367834210396, 0.08629677444696426, 0.11070671677589417, 0.16102638840675354, 0.9193330407142639]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.018054787176034242, -0.5491019027563097], "logp": -0.5671566899323439, "cache_id": 270, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 910.0875244140625, 191.54376220703125, 82.32603454589844, 32.14826965332031, 104.18109893798828, 3.421322822570801, 119.44561004638672, 266.28985595703125, 53.11229705810547, 28.380226135253906, 80.17042541503906, 72.72565460205078, 60.15878677368164, 6.017134666442871, 301.2850036621094, 250.5521240234375, 140.6173858642578, 98.43254089355469, 177.11602783203125, 31.226572036743164, 88.94405364990234, 22.733890533447266, 88.6738510131836, 104.01948547363281, 166.3539581298828, 968.7992553710938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 910.0711059570312, 191.555908203125, 82.32122802734375, 32.21305465698242, 104.21695709228516, 3.498781204223633, 119.39608001708984, 266.2431640625, 53.20726776123047, 28.415523529052734, 80.23223876953125, 72.70965576171875, 60.18403625488281, 6.103577613830566, 301.2594909667969, 250.58065795898438, 140.63023376464844, 98.3880615234375, 177.09649658203125, 31.310646057128906, 88.94184875488281, 22.805898666381836, 88.70748138427734, 104.01116943359375, 166.36610412597656, 968.7608642578125]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.8334290981292725, 0.1946035921573639, 0.08135280758142471, 0.031153298914432526, 0.09988758713006973, 0.0032973983325064182, 0.10497558116912842, 0.2571217119693756, 0.05081761255860329, 0.02899005264043808, 0.08332165330648422, 0.07370922714471817, 0.05817117914557457, 0.005859109107404947, 0.24283893406391144, 0.2524035573005676, 0.15725944936275482, 0.10922355949878693, 0.21059438586235046, 0.031038803979754448, 0.09628516435623169, 0.020176367834210396, 0.08629677444696426, 0.11070671677589417, 0.16102638840675354, 0.9193330407142639]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 18.763732614663024, "first_log_odds": 19.3125, "later_log_odds": -0.5487673853369763, "first_differing_log_odds": 19.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-main-bridge.npz", "array_sha256": "c0ce75d1d9ec2e9821fcbe62fb328ece8de24b1741709fa556582e41e4b470a9", "seconds": 0.3385461666621268, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-baseline", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 271, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.125004341563754, -0.0007876333961870428], "logp": -14.125791974959942, "cache_id": 272, "trace": []}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-19.750004341563756, -1.0109332361067545], "logp": -20.76093757767051, "cache_id": 273, "trace": []}, "log_odds": -6.635145602710569, "first_log_odds": -5.625000000000002, "later_log_odds": -1.0101456027105673, "first_differing_log_odds": -5.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-baseline.npz", "array_sha256": "eb5ca6840f06d3abd54e464c8a14970e7ba98abf9e6347196dfa63fb9da31ec5", "seconds": 1.1495137079618871, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-noop", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.376907348632812, "match_error": -9.376907348632812, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 274, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.125004341563754, -0.0007876333961870428], "logp": -14.125791974959942, "cache_id": 275, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.376907348632812, "match_error": -9.376907348632812, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-19.750004341563756, -1.0109332361067545], "logp": -20.76093757767051, "cache_id": 276, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.376907348632812, "match_error": -9.376907348632812, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.635145602710569, "first_log_odds": -5.625000000000002, "later_log_odds": -1.0101456027105673, "first_differing_log_odds": -5.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-noop.npz", "array_sha256": "09e161bbdfee2fcee5075108798e59381b6332c5bd4325c4618a83feae717bc0", "seconds": 1.1212024581618607, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-j", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.370311737060547, "actual_norm": 9.376907348632812, "relative_norm": 0.0499811307557295, "target_actual_norm": 9.376907348632812, "match_error": 0.0, "scalar": 0.1985582038496851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 277, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.500003988176882, -0.0008272071227891306], "logp": -14.500831195299671, "cache_id": 278, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.370311737060547, "actual_norm": 9.376907348632812, "relative_norm": 0.0499811307557295, "target_actual_norm": 9.376907348632812, "match_error": 0.0, "scalar": 0.1985582038496851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-19.625003988176882, -0.7666939181937837], "logp": -20.391697906370666, "cache_id": 279, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.370311737060547, "actual_norm": 9.376907348632812, "relative_norm": 0.0499811307557295, "target_actual_norm": 9.376907348632812, "match_error": 0.0, "scalar": 0.1985582038496851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.890866711070995, "first_log_odds": -5.125, "later_log_odds": -0.7658667110709949, "first_differing_log_odds": -5.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-j.npz", "array_sha256": "4d1e2787fe211aaa2798d7a2436b32f67ee0156ad9dc858a5e631623eb6d52b6", "seconds": 1.1297080828808248, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-opposite", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.366034507751465, "actual_norm": 9.376887321472168, "relative_norm": 0.04998102400623336, "target_actual_norm": 9.376907348632812, "match_error": -2.002716064453125e-05, "scalar": 9.366033249534667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 280, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.000003940246376, -0.000769327861481164], "logp": -14.000773268107858, "cache_id": 281, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.366034507751465, "actual_norm": 9.376887321472168, "relative_norm": 0.04998102400623336, "target_actual_norm": 9.376907348632812, "match_error": -2.002716064453125e-05, "scalar": 9.366033249534667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-20.250003940246376, -1.121603433289796], "logp": -21.371607373536172, "cache_id": 282, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.366034507751465, "actual_norm": 9.376887321472168, "relative_norm": 0.04998102400623336, "target_actual_norm": 9.376907348632812, "match_error": -2.002716064453125e-05, "scalar": 9.366033249534667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.3708341054283135, "first_log_odds": -6.25, "later_log_odds": -1.1208341054283135, "first_differing_log_odds": -6.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-opposite.npz", "array_sha256": "a1674cefaa246a41b44ac1118412d599d67e2f596ee52e6ce4b5c5ae4c9e1d7c", "seconds": 1.1638339580968022, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-unrelated", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.37647819519043, "actual_norm": 9.376864433288574, "relative_norm": 0.04998090200680919, "target_actual_norm": 9.376907348632812, "match_error": -4.291534423828125e-05, "scalar": 9.376478107878938, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 283, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.750002317732731, -0.0007841680214020349], "logp": -14.750786485754134, "cache_id": 284, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.37647819519043, "actual_norm": 9.376864433288574, "relative_norm": 0.04998090200680919, "target_actual_norm": 9.376907348632812, "match_error": -4.291534423828125e-05, "scalar": 9.376478107878938, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-20.250002317732733, -0.960652673375233], "logp": -21.210654991107965, "cache_id": 285, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.37647819519043, "actual_norm": 9.376864433288574, "relative_norm": 0.04998090200680919, "target_actual_norm": 9.376907348632812, "match_error": -4.291534423828125e-05, "scalar": 9.376478107878938, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.459868505353832, "first_log_odds": -5.500000000000002, "later_log_odds": -0.9598685053538301, "first_differing_log_odds": -5.500000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-unrelated.npz", "array_sha256": "0060b06355bf8ee6ac679fe90e6047429dbbfd3fc12739b4e9a7b83f7a287f6c", "seconds": 1.1625232081860304, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-ordinary", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.384061813354492, "actual_norm": 9.376980781555176, "relative_norm": 0.04998152217054869, "target_actual_norm": 9.376907348632812, "match_error": 7.343292236328125e-05, "scalar": 9.384061361197382, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 286, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-15.125002833775138, -0.0009709354188910297], "logp": -15.12597376919403, "cache_id": 287, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.384061813354492, "actual_norm": 9.376980781555176, "relative_norm": 0.04998152217054869, "target_actual_norm": 9.376907348632812, "match_error": 7.343292236328125e-05, "scalar": 9.384061361197382, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-20.000002833775138, -0.8546903838084358], "logp": -20.854693217583574, "cache_id": 288, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.384061813354492, "actual_norm": 9.376980781555176, "relative_norm": 0.04998152217054869, "target_actual_norm": 9.376907348632812, "match_error": 7.343292236328125e-05, "scalar": 9.384061361197382, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.728719448389544, "first_log_odds": -4.875, "later_log_odds": -0.8537194483895441, "first_differing_log_odds": -4.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-ordinary.npz", "array_sha256": "85e0f630263d90440305f0e2d599ff3ee436425ade98ee262bc886a83caedfc4", "seconds": 1.1624149167910218, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-random0", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.367318153381348, "actual_norm": 9.37690258026123, "relative_norm": 0.0499811053391828, "target_actual_norm": 9.376907348632812, "match_error": -4.76837158203125e-06, "scalar": 9.367320971796289, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 289, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.625002967348397, -0.0008892258256915967], "logp": -14.625892193174089, "cache_id": 290, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.367318153381348, "actual_norm": 9.37690258026123, "relative_norm": 0.0499811053391828, "target_actual_norm": 9.376907348632812, "match_error": -4.76837158203125e-06, "scalar": 9.367320971796289, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-20.500002967348397, -0.9602407138375922], "logp": -21.46024368118599, "cache_id": 291, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.367318153381348, "actual_norm": 9.37690258026123, "relative_norm": 0.0499811053391828, "target_actual_norm": 9.376907348632812, "match_error": -4.76837158203125e-06, "scalar": 9.367320971796289, "random_seed": 20271408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.834351488011901, "first_log_odds": -5.875, "later_log_odds": -0.959351488011901, "first_differing_log_odds": -5.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-random0.npz", "array_sha256": "11d38c3ac40e96b0bc820fbdd0c80a259174c35fa93fa8a62d5d85180a164027", "seconds": 1.1472004167735577, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-random1", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.369036674499512, "actual_norm": 9.376935958862305, "relative_norm": 0.0499812832550097, "target_actual_norm": 9.376907348632812, "match_error": 2.86102294921875e-05, "scalar": 9.369037934811786, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 292, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.500003568470289, -0.0007694667573762936], "logp": -14.500773035227665, "cache_id": 293, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.369036674499512, "actual_norm": 9.376935958862305, "relative_norm": 0.0499812832550097, "target_actual_norm": 9.376907348632812, "match_error": 2.86102294921875e-05, "scalar": 9.369037934811786, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-19.875003568470287, -0.9442438997585391], "logp": -20.819247468228827, "cache_id": 294, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.369036674499512, "actual_norm": 9.376935958862305, "relative_norm": 0.0499812832550097, "target_actual_norm": 9.376907348632812, "match_error": 2.86102294921875e-05, "scalar": 9.369037934811786, "random_seed": 20271409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.318474433001162, "first_log_odds": -5.374999999999998, "later_log_odds": -0.943474433001164, "first_differing_log_odds": -5.374999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-random1.npz", "array_sha256": "280356a44ab357c441f6188a1c57324f8561125942ab44b34005b84a5c945ad0", "seconds": 1.1151896673254669, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-random2", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.38070011138916, "actual_norm": 9.377004623413086, "relative_norm": 0.04998164925328219, "target_actual_norm": 9.376907348632812, "match_error": 9.72747802734375e-05, "scalar": 9.380698975292034, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 295, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-14.250003721920914, -0.0008893233725504632], "logp": -14.250893045293465, "cache_id": 296, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.38070011138916, "actual_norm": 9.377004623413086, "relative_norm": 0.04998164925328219, "target_actual_norm": 9.376907348632812, "match_error": 9.72747802734375e-05, "scalar": 9.380698975292034, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-19.875003721920912, -0.9709373512957556], "logp": -20.845941073216668, "cache_id": 297, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 187.60894775390625, "strength_one_norm": 47.19176483154297, "target_shrink_scale": 0.1985582038496851, "requested_norm": 9.38070011138916, "actual_norm": 9.377004623413086, "relative_norm": 0.04998164925328219, "target_actual_norm": 9.376907348632812, "match_error": 9.72747802734375e-05, "scalar": 9.380698975292034, "random_seed": 20271410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.5950480279232035, "first_log_odds": -5.624999999999998, "later_log_odds": -0.9700480279232053, "first_differing_log_odds": -5.624999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-random2.npz", "array_sha256": "5f0424507af620837b4ddec6065dc9579b11564b435ebaa116b32f5736846e28", "seconds": 1.1469045421108603, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-05-paraphrase-bridge", "case_id": "eval-05", "city": "Beijing", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 26549, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 958.4816284179688, 24.005619049072266, 66.93811798095703, 98.29509735107422, 103.52457427978516, 45.81254577636719, 129.2616424560547, 260.66204833984375, 53.65184783935547, 36.697303771972656, 98.0811538696289, 78.61019134521484, 10.734956741333008, 50.0142707824707, 48.70565414428711, 176.9087371826172, 28.317760467529297, 71.23106384277344, 13.40428352355957, 72.1247787475586, 140.9019012451172, 201.1528778076172, 788.4864501953125]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 958.4754638671875, 24.02292251586914, 66.85741424560547, 98.29314422607422, 103.546630859375, 45.785579681396484, 129.27586364746094, 260.54827880859375, 53.66984939575195, 36.69581985473633, 98.0879898071289, 78.6900634765625, 10.88821792602539, 50.02981948852539, 48.6824951171875, 176.92039489746094, 28.349496841430664, 71.18338775634766, 13.514573097229004, 72.06673431396484, 140.87545776367188, 201.16751098632812, 788.518310546875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.8905908465385437, 0.023000352084636688, 0.06142159551382065, 0.09296685457229614, 0.0957721546292305, 0.043986447155475616, 0.11107881367206573, 0.24473224580287933, 0.050688475370407104, 0.037201590836048126, 0.10051653534173965, 0.07837563753128052, 0.01035083644092083, 0.0526081845164299, 0.05493142455816269, 0.2042134553194046, 0.027673054486513138, 0.07591092586517334, 0.01181200984865427, 0.06844887137413025, 0.14883548021316528, 0.1937239170074463, 0.8082947134971619]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 298, "cached": true, "scoring": {"normalized": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Beijing is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 36, "end": 43}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.22668766841301, -0.0006059355972589043], "logp": -19.22729360401027, "cache_id": 299, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 958.4816284179688, 24.005619049072266, 66.93811798095703, 98.29509735107422, 103.52457427978516, 45.81254577636719, 129.2616424560547, 260.66204833984375, 53.65184783935547, 36.697303771972656, 98.0811538696289, 78.61019134521484, 10.734956741333008, 50.0142707824707, 48.70565414428711, 176.9087371826172, 28.317760467529297, 71.23106384277344, 13.40428352355957, 72.1247787475586, 140.9019012451172, 201.1528778076172, 788.4864501953125]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 958.4754638671875, 24.02292251586914, 66.85741424560547, 98.29314422607422, 103.546630859375, 45.785579681396484, 129.27586364746094, 260.54827880859375, 53.66984939575195, 36.69581985473633, 98.0879898071289, 78.6900634765625, 10.88821792602539, 50.02981948852539, 48.6824951171875, 176.92039489746094, 28.349496841430664, 71.18338775634766, 13.514573097229004, 72.06673431396484, 140.87545776367188, 201.16751098632812, 788.518310546875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.8905908465385437, 0.023000352084636688, 0.06142159551382065, 0.09296685457229614, 0.0957721546292305, 0.043986447155475616, 0.11107881367206573, 0.24473224580287933, 0.050688475370407104, 0.037201590836048126, 0.10051653534173965, 0.07837563753128052, 0.01035083644092083, 0.0526081845164299, 0.05493142455816269, 0.2042134553194046, 0.027673054486513138, 0.07591092586517334, 0.01181200984865427, 0.06844887137413025, 0.14883548021316528, 0.1937239170074463, 0.8082947134971619]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-7.851687668413011, -0.40855380974042516], "logp": -8.260241478153436, "cache_id": 300, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 958.4816284179688, 24.005619049072266, 66.93811798095703, 98.29509735107422, 103.52457427978516, 45.81254577636719, 129.2616424560547, 260.66204833984375, 53.65184783935547, 36.697303771972656, 98.0811538696289, 78.61019134521484, 10.734956741333008, 50.0142707824707, 48.70565414428711, 176.9087371826172, 28.317760467529297, 71.23106384277344, 13.40428352355957, 72.1247787475586, 140.9019012451172, 201.1528778076172, 788.4864501953125]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 958.4754638671875, 24.02292251586914, 66.85741424560547, 98.29314422607422, 103.546630859375, 45.785579681396484, 129.27586364746094, 260.54827880859375, 53.66984939575195, 36.69581985473633, 98.0879898071289, 78.6900634765625, 10.88821792602539, 50.02981948852539, 48.6824951171875, 176.92039489746094, 28.349496841430664, 71.18338775634766, 13.514573097229004, 72.06673431396484, 140.87545776367188, 201.16751098632812, 788.518310546875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.8905908465385437, 0.023000352084636688, 0.06142159551382065, 0.09296685457229614, 0.0957721546292305, 0.043986447155475616, 0.11107881367206573, 0.24473224580287933, 0.050688475370407104, 0.037201590836048126, 0.10051653534173965, 0.07837563753128052, 0.01035083644092083, 0.0526081845164299, 0.05493142455816269, 0.2042134553194046, 0.027673054486513138, 0.07591092586517334, 0.01181200984865427, 0.06844887137413025, 0.14883548021316528, 0.1937239170074463, 0.8082947134971619]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 10.967052125856835, "first_log_odds": 11.375, "later_log_odds": -0.407947874143165, "first_differing_log_odds": 11.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-05-paraphrase-bridge.npz", "array_sha256": "b3bada2cd8b843712326ff38321e9dc220d1fc4be5c26914622e4ac28c7fe551", "seconds": 1.1499468330293894, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-baseline", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 301, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-1.7873008848872562e-05, -0.0011031764114334496], "logp": -0.0011210494202823222, "cache_id": 302, "trace": []}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.375017873008847, -1.374775795485735], "logp": -28.749793668494583, "cache_id": 303, "trace": []}, "log_odds": -28.748672619074302, "first_log_odds": -27.375, "later_log_odds": -1.373672619074302, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-baseline.npz", "array_sha256": "15e93adb608af9996828485c772928b7b3324f1bf98f59af9401a483f89aea75", "seconds": 0.3742239591665566, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-noop", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.458961486816406, "match_error": -9.458961486816406, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 304, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-1.7873008848872562e-05, -0.0011031764114334496], "logp": -0.0011210494202823222, "cache_id": 305, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.458961486816406, "match_error": -9.458961486816406, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.375017873008847, -1.374775795485735], "logp": -28.749793668494583, "cache_id": 306, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.458961486816406, "match_error": -9.458961486816406, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.748672619074302, "first_log_odds": -27.375, "later_log_odds": -1.373672619074302, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-noop.npz", "array_sha256": "5b8a858a5137e6977f27bb4f6ac558245c778b1bb417b7d063dccf4fe7630422", "seconds": 0.3362044580280781, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-j", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457220077514648, "actual_norm": 9.458961486816406, "relative_norm": 0.04999651985621643, "target_actual_norm": 9.458961486816406, "match_error": 0.0, "scalar": 0.2719940095639529, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 307, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-2.1292330327613687e-05, -0.0016705196753870296], "logp": -0.0016918120057146432, "cache_id": 308, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457220077514648, "actual_norm": 9.458961486816406, "relative_norm": 0.04999651985621643, "target_actual_norm": 9.458961486816406, "match_error": 0.0, "scalar": 0.2719940095639529, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.750021292330327, -1.109192527070415], "logp": -28.859213819400743, "cache_id": 309, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457220077514648, "actual_norm": 9.458961486816406, "relative_norm": 0.04999651985621643, "target_actual_norm": 9.458961486816406, "match_error": 0.0, "scalar": 0.2719940095639529, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.85752200739503, "first_log_odds": -27.75, "later_log_odds": -1.1075220073950298, "first_differing_log_odds": -27.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-j.npz", "array_sha256": "891479fc063244d38a56e7e02c10a8c93e8e2e32a7659165f8e659aad94ec344", "seconds": 0.3388807917945087, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-opposite", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.456940650939941, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.45694083464332, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 310, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-2.284633935109651e-05, -0.0010566704623773697], "logp": -0.0010795168017284662, "cache_id": 311, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.456940650939941, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.45694083464332, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.375022846339352, -1.5130515975930836], "logp": -28.888074443932435, "cache_id": 312, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.456940650939941, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.45694083464332, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.886994927130708, "first_log_odds": -27.375, "later_log_odds": -1.5119949271307078, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-opposite.npz", "array_sha256": "8ece601004dc6570b6e8a1b587435cc3fdc2a39fbdc240d5f8e13e5265b92fb1", "seconds": 0.33943958301097155, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-unrelated", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.463435173034668, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.463435788056813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 313, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-1.8910750568631424e-05, -0.0012515586853786818], "logp": -0.0012704694359473132, "cache_id": 314, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.463435173034668, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.463435788056813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.62501891075057, -1.4102410464979889], "logp": -29.03525995724856, "cache_id": 315, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.463435173034668, "actual_norm": 9.458719253540039, "relative_norm": 0.049995239502043445, "target_actual_norm": 9.458961486816406, "match_error": -0.0002422332763671875, "scalar": 9.463435788056813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.03398948781261, "first_log_odds": -27.625, "later_log_odds": -1.4089894878126117, "first_differing_log_odds": -27.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-unrelated.npz", "array_sha256": "a69c876cf598928919c96d74d5313617efd8f174706cd8901f2c79a7c514e504", "seconds": 0.34062787471339107, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-ordinary", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.461991310119629, "actual_norm": 9.458999633789062, "relative_norm": 0.04999672148679485, "target_actual_norm": 9.458961486816406, "match_error": 3.814697265625e-05, "scalar": 9.461992465076037, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 316, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-2.1484942515650117e-05, -0.0013558234087464693], "logp": -0.0013773083512621195, "cache_id": 317, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.461991310119629, "actual_norm": 9.458999633789062, "relative_norm": 0.04999672148679485, "target_actual_norm": 9.458961486816406, "match_error": 3.814697265625e-05, "scalar": 9.461992465076037, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.375021484942515, -1.2314918006014937], "logp": -28.606513285544008, "cache_id": 318, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.461991310119629, "actual_norm": 9.458999633789062, "relative_norm": 0.04999672148679485, "target_actual_norm": 9.458961486816406, "match_error": 3.814697265625e-05, "scalar": 9.461992465076037, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.605135977192745, "first_log_odds": -27.375, "later_log_odds": -1.2301359771927451, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-ordinary.npz", "array_sha256": "5e782a45fd1f35cb0843c2f19b5d48ff406dbaf9591bbb26d24a10fa189832b4", "seconds": 0.3386056669987738, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-random0", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.452502250671387, "actual_norm": 9.458891868591309, "relative_norm": 0.04999615188041081, "target_actual_norm": 9.458961486816406, "match_error": -6.961822509765625e-05, "scalar": 9.452502616477432, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 319, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-1.906752856772538e-05, -0.0012301292155648283], "logp": -0.0012491967441325537, "cache_id": 320, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.452502250671387, "actual_norm": 9.458891868591309, "relative_norm": 0.04999615188041081, "target_actual_norm": 9.458961486816406, "match_error": -6.961822509765625e-05, "scalar": 9.452502616477432, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.37501906752857, -1.441142910643214], "logp": -28.816161978171785, "cache_id": 321, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.452502250671387, "actual_norm": 9.458891868591309, "relative_norm": 0.04999615188041081, "target_actual_norm": 9.458961486816406, "match_error": -6.961822509765625e-05, "scalar": 9.452502616477432, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.81491278142765, "first_log_odds": -27.375, "later_log_odds": -1.4399127814276511, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-random0.npz", "array_sha256": "170cf54b61f4821d367f354e6395a74f916c2879a1ac1b55730e81c5dd6f6e8d", "seconds": 0.3375893752090633, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-random1", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.438214302062988, "actual_norm": 9.458958625793457, "relative_norm": 0.049996504733923045, "target_actual_norm": 9.458961486816406, "match_error": -2.86102294921875e-06, "scalar": 9.438213718967745, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 322, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-2.1609192164611442e-05, -0.0012472178517244764], "logp": -0.0012688270438890878, "cache_id": 323, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.438214302062988, "actual_norm": 9.458958625793457, "relative_norm": 0.049996504733923045, "target_actual_norm": 9.458961486816406, "match_error": -2.86102294921875e-06, "scalar": 9.438213718967745, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.375021609192164, -1.2476979471545602], "logp": -28.622719556346723, "cache_id": 324, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.438214302062988, "actual_norm": 9.458958625793457, "relative_norm": 0.049996504733923045, "target_actual_norm": 9.458961486816406, "match_error": -2.86102294921875e-06, "scalar": 9.438213718967745, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.621450729302833, "first_log_odds": -27.375, "later_log_odds": -1.2464507293028326, "first_differing_log_odds": -27.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-random1.npz", "array_sha256": "65be041adfc1785b4850b1b0bef2bd67326026be11ca74421a990c9a881c58fe", "seconds": 0.3381428332068026, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-random2", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CNY<|im_end|>", "output_text": "CNY", "output_ids": [34, 23054, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457245826721191, "actual_norm": 9.45888614654541, "relative_norm": 0.049996121635824044, "target_actual_norm": 9.458961486816406, "match_error": -7.534027099609375e-05, "scalar": 9.457247540776734, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 325, "cached": true, "scoring": {"normalized": "CNY", "bare_format": true, "malformed": false, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-1.8770290033031342e-05, -0.0012158603877489612], "logp": -0.0012346306777819927, "cache_id": 326, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457245826721191, "actual_norm": 9.45888614654541, "relative_norm": 0.049996121635824044, "target_actual_norm": 9.458961486816406, "match_error": -7.534027099609375e-05, "scalar": 9.457247540776734, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-27.625018770290033, -1.2962267318246046], "logp": -28.921245502114637, "cache_id": 327, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 189.19239807128906, "strength_one_norm": 34.769954681396484, "target_shrink_scale": 0.2719940095639529, "requested_norm": 9.457245826721191, "actual_norm": 9.45888614654541, "relative_norm": 0.049996121635824044, "target_actual_norm": 9.458961486816406, "match_error": -7.534027099609375e-05, "scalar": 9.457247540776734, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.920010871436855, "first_log_odds": -27.625, "later_log_odds": -1.2950108714368547, "first_differing_log_odds": -27.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-random2.npz", "array_sha256": "c79d1962327f3c98e15d8631958436fffd2c58a5427e57452ac704378ee185fc", "seconds": 0.33750808285549283, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-main-bridge", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 591.3673706054688, 838.0867309570312, 224.05577087402344, 83.50550079345703, 39.13447570800781, 92.58072662353516, 22.465286254882812, 118.56633758544922, 266.91253662109375, 43.85993576049805, 30.01070785522461, 82.51996612548828, 83.95121002197266, 51.22251892089844, 25.02232551574707, 322.6773986816406, 264.6938171386719, 134.75904846191406, 107.41509246826172, 169.9459686279297, 29.405092239379883, 67.97724914550781, 14.269868850708008, 86.25007629394531, 95.120849609375, 164.76834106445312, 1052.801513671875]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 591.3821411132812, 838.0656127929688, 224.0479736328125, 83.5694580078125, 39.181732177734375, 92.59700012207031, 22.529394149780273, 118.51901245117188, 266.90106201171875, 43.85299301147461, 29.992698669433594, 82.52882385253906, 84.01775360107422, 51.22805404663086, 25.06598663330078, 322.6439514160156, 264.6921081542969, 134.8207550048828, 107.3930435180664, 169.95623779296875, 29.470504760742188, 68.02218627929688, 14.368876457214355, 86.29251098632812, 95.12559509277344, 164.7175750732422, 1052.807373046875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.5049098134040833, 0.8406094908714294, 0.23166386783123016, 0.08033637702465057, 0.03763629123568535, 0.08824770897626877, 0.021384194493293762, 0.10443373769521713, 0.2601248025894165, 0.04208784177899361, 0.03056594543159008, 0.08596225827932358, 0.08558680862188339, 0.04944333806633949, 0.023849347606301308, 0.25926193594932556, 0.26309525966644287, 0.1505756378173828, 0.11979590356349945, 0.2049625664949417, 0.029355553910136223, 0.07306501269340515, 0.012751886621117592, 0.08418115228414536, 0.10281798988580704, 0.1592414677143097, 0.983086347579956]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 328, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-17.107742400936626, -0.001299125790798949], "logp": -17.109041526727424, "cache_id": 329, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 591.3673706054688, 838.0867309570312, 224.05577087402344, 83.50550079345703, 39.13447570800781, 92.58072662353516, 22.465286254882812, 118.56633758544922, 266.91253662109375, 43.85993576049805, 30.01070785522461, 82.51996612548828, 83.95121002197266, 51.22251892089844, 25.02232551574707, 322.6773986816406, 264.6938171386719, 134.75904846191406, 107.41509246826172, 169.9459686279297, 29.405092239379883, 67.97724914550781, 14.269868850708008, 86.25007629394531, 95.120849609375, 164.76834106445312, 1052.801513671875]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 591.3821411132812, 838.0656127929688, 224.0479736328125, 83.5694580078125, 39.181732177734375, 92.59700012207031, 22.529394149780273, 118.51901245117188, 266.90106201171875, 43.85299301147461, 29.992698669433594, 82.52882385253906, 84.01775360107422, 51.22805404663086, 25.06598663330078, 322.6439514160156, 264.6921081542969, 134.8207550048828, 107.3930435180664, 169.95623779296875, 29.470504760742188, 68.02218627929688, 14.368876457214355, 86.29251098632812, 95.12559509277344, 164.7175750732422, 1052.807373046875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.5049098134040833, 0.8406094908714294, 0.23166386783123016, 0.08033637702465057, 0.03763629123568535, 0.08824770897626877, 0.021384194493293762, 0.10443373769521713, 0.2601248025894165, 0.04208784177899361, 0.03056594543159008, 0.08596225827932358, 0.08558680862188339, 0.04944333806633949, 0.023849347606301308, 0.25926193594932556, 0.26309525966644287, 0.1505756378173828, 0.11979590356349945, 0.2049625664949417, 0.029355553910136223, 0.07306501269340515, 0.012751886621117592, 0.08418115228414536, 0.10281798988580704, 0.1592414677143097, 0.983086347579956]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.7327424009366275, -0.6022932815114506], "logp": -1.335035682448078, "cache_id": 330, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 63.38279724121094, 12.71402359008789, 177.2279052734375, 145.96441650390625, 108.21319580078125, 591.3673706054688, 838.0867309570312, 224.05577087402344, 83.50550079345703, 39.13447570800781, 92.58072662353516, 22.465286254882812, 118.56633758544922, 266.91253662109375, 43.85993576049805, 30.01070785522461, 82.51996612548828, 83.95121002197266, 51.22251892089844, 25.02232551574707, 322.6773986816406, 264.6938171386719, 134.75904846191406, 107.41509246826172, 169.9459686279297, 29.405092239379883, 67.97724914550781, 14.269868850708008, 86.25007629394531, 95.120849609375, 164.76834106445312, 1052.801513671875]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 63.413944244384766, 12.742715835571289, 177.18084716796875, 145.90260314941406, 108.21171569824219, 591.3821411132812, 838.0656127929688, 224.0479736328125, 83.5694580078125, 39.181732177734375, 92.59700012207031, 22.529394149780273, 118.51901245117188, 266.90106201171875, 43.85299301147461, 29.992698669433594, 82.52882385253906, 84.01775360107422, 51.22805404663086, 25.06598663330078, 322.6439514160156, 264.6921081542969, 134.8207550048828, 107.3930435180664, 169.95623779296875, 29.470504760742188, 68.02218627929688, 14.368876457214355, 86.29251098632812, 95.12559509277344, 164.7175750732422, 1052.807373046875]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.05821176990866661, 0.011180221103131771, 0.14414602518081665, 0.12488099187612534, 0.09535624831914902, 0.5049098134040833, 0.8406094908714294, 0.23166386783123016, 0.08033637702465057, 0.03763629123568535, 0.08824770897626877, 0.021384194493293762, 0.10443373769521713, 0.2601248025894165, 0.04208784177899361, 0.03056594543159008, 0.08596225827932358, 0.08558680862188339, 0.04944333806633949, 0.023849347606301308, 0.25926193594932556, 0.26309525966644287, 0.1505756378173828, 0.11979590356349945, 0.2049625664949417, 0.029355553910136223, 0.07306501269340515, 0.012751886621117592, 0.08418115228414536, 0.10281798988580704, 0.1592414677143097, 0.983086347579956]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.774005844279346, "first_log_odds": 16.375, "later_log_odds": -0.6009941557206542, "first_differing_log_odds": 16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-main-bridge.npz", "array_sha256": "a07359fb5cd5953caa9bff8a4f01ecde274b726252443812a3d551e7ffc5bcd3", "seconds": 0.3371121250092983, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-baseline", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 331, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.625000022457687, -0.0006544175094724378], "logp": -19.62565443996716, "cache_id": 332, "trace": []}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.250000022457687, -1.8015233314839714], "logp": -31.051523353941658, "cache_id": 333, "trace": []}, "log_odds": -11.425868913974497, "first_log_odds": -9.625, "later_log_odds": -1.8008689139744973, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-baseline.npz", "array_sha256": "311681ffd11538bd064536c088923724226935dcea260b8bf772e56fba5036ad", "seconds": 1.1921765827573836, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-noop", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.374737739562988, "match_error": -9.374737739562988, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 334, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.625000022457687, -0.0006544175094724378], "logp": -19.62565443996716, "cache_id": 335, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.374737739562988, "match_error": -9.374737739562988, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.250000022457687, -1.8015233314839714], "logp": -31.051523353941658, "cache_id": 336, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.374737739562988, "match_error": -9.374737739562988, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.425868913974497, "first_log_odds": -9.625, "later_log_odds": -1.8008689139744973, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-noop.npz", "array_sha256": "3a529e2b8c07da89f12b621425c130e7ae24144ea21b7143f5d32b05aa337b35", "seconds": 1.179050332866609, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-j", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.366251945495605, "actual_norm": 9.374737739562988, "relative_norm": 0.049997950262691895, "target_actual_norm": 9.374737739562988, "match_error": 0.0, "scalar": 0.24990897623709174, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 337, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.375000025515185, -0.0007318709385867649], "logp": -19.37573189645377, "cache_id": 338, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.366251945495605, "actual_norm": 9.374737739562988, "relative_norm": 0.049997950262691895, "target_actual_norm": 9.374737739562988, "match_error": 0.0, "scalar": 0.24990897623709174, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.000000025515185, -1.5460601572739798], "logp": -30.546060182789166, "cache_id": 339, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.366251945495605, "actual_norm": 9.374737739562988, "relative_norm": 0.049997950262691895, "target_actual_norm": 9.374737739562988, "match_error": 0.0, "scalar": 0.24990897623709174, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.170328286335394, "first_log_odds": -9.625, "later_log_odds": -1.545328286335394, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-j.npz", "array_sha256": "865d3694cb452ae0fe3f965c8bbf1957d406d8a80186f269e14627ef77ec6077", "seconds": 1.1864920831285417, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-opposite", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.36637020111084, "actual_norm": 9.374701499938965, "relative_norm": 0.04999775698721371, "target_actual_norm": 9.374737739562988, "match_error": -3.62396240234375e-05, "scalar": 9.36636948154046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 340, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.250000032957214, -0.000669466296300207], "logp": -19.250669499253515, "cache_id": 341, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.36637020111084, "actual_norm": 9.374701499938965, "relative_norm": 0.04999775698721371, "target_actual_norm": 9.374737739562988, "match_error": -3.62396240234375e-05, "scalar": 9.36636948154046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-28.875000032957214, -1.929371882769472], "logp": -30.804371915726687, "cache_id": 342, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.36637020111084, "actual_norm": 9.374701499938965, "relative_norm": 0.04999775698721371, "target_actual_norm": 9.374737739562988, "match_error": -3.62396240234375e-05, "scalar": 9.36636948154046, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.553702416473172, "first_log_odds": -9.625, "later_log_odds": -1.9287024164731719, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-opposite.npz", "array_sha256": "4a46cecc215e9c85c7a303621d76a65549ab9715b3a181868c3f5cc90a535ce5", "seconds": 1.1971589159220457, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-unrelated", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.357605934143066, "actual_norm": 9.3742094039917, "relative_norm": 0.04999513250966784, "target_actual_norm": 9.374737739562988, "match_error": -0.0005283355712890625, "scalar": 9.3576078438673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 343, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.500000026493176, -0.000662323609273462], "logp": -19.50066235010245, "cache_id": 344, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.357605934143066, "actual_norm": 9.3742094039917, "relative_norm": 0.04999513250966784, "target_actual_norm": 9.374737739562988, "match_error": -0.0005283355712890625, "scalar": 9.3576078438673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.125000026493176, -1.8956252875829054], "logp": -31.02062531407608, "cache_id": 345, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.357605934143066, "actual_norm": 9.3742094039917, "relative_norm": 0.04999513250966784, "target_actual_norm": 9.374737739562988, "match_error": -0.0005283355712890625, "scalar": 9.3576078438673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.519962963973633, "first_log_odds": -9.625, "later_log_odds": -1.894962963973633, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-unrelated.npz", "array_sha256": "c7f056b7252d4b5970e5d67194cacb81270d62b794a7af8644949b3dec080294", "seconds": 1.1777695827186108, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-ordinary", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.370285987854004, "actual_norm": 9.374744415283203, "relative_norm": 0.049997985866069454, "target_actual_norm": 9.374737739562988, "match_error": 6.67572021484375e-06, "scalar": 9.370285397153566, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 346, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.625000023341215, -0.0007168663385776026], "logp": -19.625716889679794, "cache_id": 347, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.370285987854004, "actual_norm": 9.374744415283203, "relative_norm": 0.049997985866069454, "target_actual_norm": 9.374737739562988, "match_error": 6.67572021484375e-06, "scalar": 9.370285397153566, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.125000023341215, -1.4265786366453512], "logp": -30.551578659986568, "cache_id": 348, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.370285987854004, "actual_norm": 9.374744415283203, "relative_norm": 0.049997985866069454, "target_actual_norm": 9.374737739562988, "match_error": 6.67572021484375e-06, "scalar": 9.370285397153566, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.925861770306774, "first_log_odds": -9.5, "later_log_odds": -1.4258617703067742, "first_differing_log_odds": -9.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-ordinary.npz", "array_sha256": "2895b7957575709fb59efd5db3dbacb4eec53f91c6d456590ceba1566f0cffbe", "seconds": 1.2151407501660287, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-random0", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.372771263122559, "actual_norm": 9.374747276306152, "relative_norm": 0.04999800112465984, "target_actual_norm": 9.374737739562988, "match_error": 9.5367431640625e-06, "scalar": 9.37277084130983, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 349, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-18.875000034833437, -0.0006454863909648611], "logp": -18.875645521224403, "cache_id": 350, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.372771263122559, "actual_norm": 9.374747276306152, "relative_norm": 0.04999800112465984, "target_actual_norm": 9.374737739562988, "match_error": 9.5367431640625e-06, "scalar": 9.37277084130983, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.000000034833437, -1.9080051187884273], "logp": -30.908005153621865, "cache_id": 351, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.372771263122559, "actual_norm": 9.374747276306152, "relative_norm": 0.04999800112465984, "target_actual_norm": 9.374737739562988, "match_error": 9.5367431640625e-06, "scalar": 9.37277084130983, "random_seed": 20271508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.032359632397462, "first_log_odds": -10.125, "later_log_odds": -1.907359632397462, "first_differing_log_odds": -10.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-random0.npz", "array_sha256": "af2c87e1538a4a43a640a5f70aced4bd62a39dc2992711ac02344ccb77a6ac60", "seconds": 1.1969237918965518, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-random1", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.367477416992188, "actual_norm": 9.37468147277832, "relative_norm": 0.04999765017708103, "target_actual_norm": 9.374737739562988, "match_error": -5.626678466796875e-05, "scalar": 9.367478096919513, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 352, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-20.00000002000528, -0.000586968143703187], "logp": -20.000586988148985, "cache_id": 353, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.367477416992188, "actual_norm": 9.37468147277832, "relative_norm": 0.04999765017708103, "target_actual_norm": 9.374737739562988, "match_error": -5.626678466796875e-05, "scalar": 9.367478096919513, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.37500002000528, -1.8431789813715946], "logp": -31.218179001376875, "cache_id": 354, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.367477416992188, "actual_norm": 9.37468147277832, "relative_norm": 0.04999765017708103, "target_actual_norm": 9.374737739562988, "match_error": -5.626678466796875e-05, "scalar": 9.367478096919513, "random_seed": 20271509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.21759201322789, "first_log_odds": -9.375, "later_log_odds": -1.8425920132278897, "first_differing_log_odds": -9.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-random1.npz", "array_sha256": "9f015735bd4f531e66c3ed46c8340fd102914d79e2ee4d2e35a9c710943140ae", "seconds": 1.178766124881804, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-random2", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.374004364013672, "actual_norm": 9.374652862548828, "relative_norm": 0.0499974975911772, "target_actual_norm": 9.374737739562988, "match_error": -8.487701416015625e-05, "scalar": 9.374004622941357, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 355, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-20.12500001759791, -0.0007198043455263178], "logp": -20.125719821943434, "cache_id": 356, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.374004364013672, "actual_norm": 9.374652862548828, "relative_norm": 0.0499974975911772, "target_actual_norm": 9.374737739562988, "match_error": -8.487701416015625e-05, "scalar": 9.374004622941357, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.37500001759791, -1.7904821222211893], "logp": -31.165482139819098, "cache_id": 357, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 187.50244140625, "strength_one_norm": 37.47865295410156, "target_shrink_scale": 0.24990897623709174, "requested_norm": 9.374004364013672, "actual_norm": 9.374652862548828, "relative_norm": 0.0499974975911772, "target_actual_norm": 9.374737739562988, "match_error": -8.487701416015625e-05, "scalar": 9.374004622941357, "random_seed": 20271510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.039762317875663, "first_log_odds": -9.25, "later_log_odds": -1.7897623178756632, "first_differing_log_odds": -9.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-random2.npz", "array_sha256": "b47f9ad21a5aa394155d6d93aeb51bc3d525be29faeeb4317af21d3d15a859eb", "seconds": 1.196375624742359, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-06-paraphrase-bridge", "case_id": "eval-06", "city": "Wuhan", "country": "China", "code": "CNY", "target_country": "Brazil", "target_code": "BRL", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.<|im_end|>", "output_text": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "output_ids": [785, 11413, 1483, 304, 36130, 9917, 374, 279, 8453, 72283, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 34, 23054, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 608.680908203125, 879.2656860351562, 53.827762603759766, 127.76602935791016, 176.81204223632812, 133.48983764648438, 62.46426010131836, 117.01082611083984, 267.7257080078125, 55.86729049682617, 42.11574935913086, 96.82421875, 85.39269256591797, 11.673822402954102, 24.968128204345703, 62.08768081665039, 180.90167236328125, 26.313720703125, 71.6764144897461, 3.471674680709839, 69.71295928955078, 130.63540649414062, 179.21437072753906, 772.6361694335938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 608.6808471679688, 879.3463745117188, 53.831729888916016, 127.75669860839844, 176.8099365234375, 133.50674438476562, 62.47325134277344, 117.03787994384766, 267.72265625, 55.85395431518555, 42.10332107543945, 96.83452606201172, 85.43402862548828, 11.724827766418457, 24.957334518432617, 62.145843505859375, 180.9232635498047, 26.413726806640625, 71.67926788330078, 3.5738468170166016, 69.67301940917969, 130.71983337402344, 179.23934936523438, 772.6034545898438]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.5182290077209473, 0.8659197092056274, 0.05288325622677803, 0.11749514937400818, 0.1657865345478058, 0.12247374653816223, 0.060367316007614136, 0.10015016794204712, 0.2523081302642822, 0.052797138690948486, 0.042607370764017105, 0.09936321526765823, 0.0848352387547493, 0.011066646315157413, 0.026391571387648582, 0.07032786309719086, 0.20762521028518677, 0.025804642587900162, 0.07638493925333023, 0.0031416993588209152, 0.06621898710727692, 0.137148916721344, 0.17280444502830505, 0.8122639656066895]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 358, "cached": true, "scoring": {"normalized": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **CNY**.", "bare_format": false, "malformed": true, "first_code": "CNY", "all_codes": ["CNY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Wuhan is the Chinese yuan, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Chinese", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [34, 23054], "tokens": ["C", "NY"], "log_probs": [-19.018605934460616, -0.00041256482450042293], "logp": -19.019018499285117, "cache_id": 359, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 608.680908203125, 879.2656860351562, 53.827762603759766, 127.76602935791016, 176.81204223632812, 133.48983764648438, 62.46426010131836, 117.01082611083984, 267.7257080078125, 55.86729049682617, 42.11574935913086, 96.82421875, 85.39269256591797, 11.673822402954102, 24.968128204345703, 62.08768081665039, 180.90167236328125, 26.313720703125, 71.6764144897461, 3.471674680709839, 69.71295928955078, 130.63540649414062, 179.21437072753906, 772.6361694335938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 608.6808471679688, 879.3463745117188, 53.831729888916016, 127.75669860839844, 176.8099365234375, 133.50674438476562, 62.47325134277344, 117.03787994384766, 267.72265625, 55.85395431518555, 42.10332107543945, 96.83452606201172, 85.43402862548828, 11.724827766418457, 24.957334518432617, 62.145843505859375, 180.9232635498047, 26.413726806640625, 71.67926788330078, 3.5738468170166016, 69.67301940917969, 130.71983337402344, 179.23934936523438, 772.6034545898438]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.5182290077209473, 0.8659197092056274, 0.05288325622677803, 0.11749514937400818, 0.1657865345478058, 0.12247374653816223, 0.060367316007614136, 0.10015016794204712, 0.2523081302642822, 0.052797138690948486, 0.042607370764017105, 0.09936321526765823, 0.0848352387547493, 0.011066646315157413, 0.026391571387648582, 0.07032786309719086, 0.20762521028518677, 0.025804642587900162, 0.07638493925333023, 0.0031416993588209152, 0.06621898710727692, 0.137148916721344, 0.17280444502830505, 0.8122639656066895]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-9.893605934460615, -1.0106961806048522], "logp": -10.904302115065466, "cache_id": 360, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[65.10883331298828, 33.17898941040039, 95.409423828125, 80.20065307617188, 31.537418365478516, 5.664115905761719, 157.2963409423828, 156.83103942871094, 608.680908203125, 879.2656860351562, 53.827762603759766, 127.76602935791016, 176.81204223632812, 133.48983764648438, 62.46426010131836, 117.01082611083984, 267.7257080078125, 55.86729049682617, 42.11574935913086, 96.82421875, 85.39269256591797, 11.673822402954102, 24.968128204345703, 62.08768081665039, 180.90167236328125, 26.313720703125, 71.6764144897461, 3.471674680709839, 69.71295928955078, 130.63540649414062, 179.21437072753906, 772.6361694335938]], "actual_norm": [[65.12982177734375, 33.19666290283203, 95.37474060058594, 80.17506408691406, 31.53629493713379, 5.792428970336914, 157.27621459960938, 156.82980346679688, 608.6808471679688, 879.3463745117188, 53.831729888916016, 127.75669860839844, 176.8099365234375, 133.50674438476562, 62.47325134277344, 117.03787994384766, 267.72265625, 55.85395431518555, 42.10332107543945, 96.83452606201172, 85.43402862548828, 11.724827766418457, 24.957334518432617, 62.145843505859375, 180.9232635498047, 26.413726806640625, 71.67926788330078, 3.5738468170166016, 69.67301940917969, 130.71983337402344, 179.23934936523438, 772.6034545898438]], "relative_norm": [[0.003801293671131134, 0.035214316099882126, 0.10551419854164124, 0.06990427523851395, 0.028706707060337067, 0.005032091401517391, 0.13987968862056732, 0.13931289315223694, 0.5182290077209473, 0.8659197092056274, 0.05288325622677803, 0.11749514937400818, 0.1657865345478058, 0.12247374653816223, 0.060367316007614136, 0.10015016794204712, 0.2523081302642822, 0.052797138690948486, 0.042607370764017105, 0.09936321526765823, 0.0848352387547493, 0.011066646315157413, 0.026391571387648582, 0.07032786309719086, 0.20762521028518677, 0.025804642587900162, 0.07638493925333023, 0.0031416993588209152, 0.06621898710727692, 0.137148916721344, 0.17280444502830505, 0.8122639656066895]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 8.114716384219651, "first_log_odds": 9.125000000000002, "later_log_odds": -1.0102836157803505, "first_differing_log_odds": 9.125000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-06-paraphrase-bridge.npz", "array_sha256": "76c1a747fe102625a0822d9c2a6dde8d64621463e891d004b2a4a723504cdd47", "seconds": 1.1955347079783678, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-baseline", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 361, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00034564298028109285, -0.0004956633878024187], "logp": -0.0008413063680835116, "cache_id": 362, "trace": []}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.37534564298028, -3.615618810096087e-06], "logp": -17.37534925859909, "cache_id": 363, "trace": []}, "log_odds": -17.374507952231006, "first_log_odds": -17.375, "later_log_odds": 0.0004920477689935865, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-baseline.npz", "array_sha256": "10aa006c7c7e3e2799c1bb62506277b7216b7d0f0308c4525697901191b4788f", "seconds": 0.38508458295837045, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456682496, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-noop", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.21264362335205, "match_error": -9.21264362335205, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 364, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00034564298028109285, -0.0004956633878024187], "logp": -0.0008413063680835116, "cache_id": 365, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.21264362335205, "match_error": -9.21264362335205, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.37534564298028, -3.615618810096087e-06], "logp": -17.37534925859909, "cache_id": 366, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.21264362335205, "match_error": -9.21264362335205, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.374507952231006, "first_log_odds": -17.375, "later_log_odds": 0.0004920477689935865, "first_differing_log_odds": -17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-noop.npz", "array_sha256": "91450cc9b9ae6d0faecc92f876f3fb433f6f77fdb4cd7b20e45ccd5738221c97", "seconds": 0.3342261668294668, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-j", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.223411560058594, "actual_norm": 9.21264362335205, "relative_norm": 0.049941631279339455, "target_actual_norm": 9.21264362335205, "match_error": 0.0, "scalar": 0.9767441616204054, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 367, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0006917224112304799, -0.0005571507252943171], "logp": -0.0012488731365247968, "cache_id": 368, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.223411560058594, "actual_norm": 9.21264362335205, "relative_norm": 0.049941631279339455, "target_actual_norm": 9.21264362335205, "match_error": 0.0, "scalar": 0.9767441616204054, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-15.750691722411231, -2.5094751029091014e-06], "logp": -15.750694231886333, "cache_id": 369, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.223411560058594, "actual_norm": 9.21264362335205, "relative_norm": 0.049941631279339455, "target_actual_norm": 9.21264362335205, "match_error": 0.0, "scalar": 0.9767441616204054, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.749445358749808, "first_log_odds": -15.75, "later_log_odds": 0.0005546412501917786, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-j.npz", "array_sha256": "f6af06ef3711a24fba518703407c1d4a402a024be16e169942139eaeb4fd58e7", "seconds": 0.33202475029975176, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-opposite", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216087341308594, "actual_norm": 9.213677406311035, "relative_norm": 0.049947235404438746, "target_actual_norm": 9.21264362335205, "match_error": 0.001033782958984375, "scalar": 9.21608768110309, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 370, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00029983799447279075, -0.00037699221352262003], "logp": -0.0006768302079954108, "cache_id": 371, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216087341308594, "actual_norm": 9.213677406311035, "relative_norm": 0.049947235404438746, "target_actual_norm": 9.21264362335205, "match_error": 0.001033782958984375, "scalar": 9.21608768110309, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-18.000299837994472, -3.040973233319004e-06], "logp": -18.000302878967705, "cache_id": 372, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216087341308594, "actual_norm": 9.213677406311035, "relative_norm": 0.049947235404438746, "target_actual_norm": 9.21264362335205, "match_error": 0.001033782958984375, "scalar": 9.21608768110309, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.99962604875971, "first_log_odds": -18.0, "later_log_odds": 0.0003739512402916034, "first_differing_log_odds": -18.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-opposite.npz", "array_sha256": "7338b8e274d530a4587c1627662377453c99b02d38f9f47ad3ccc85a02e032fc", "seconds": 0.33613175014033914, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-unrelated", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.209342956542969, "actual_norm": 9.212175369262695, "relative_norm": 0.049939092879502045, "target_actual_norm": 9.21264362335205, "match_error": -0.00046825408935546875, "scalar": 9.209340139386768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 373, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00042199657636707457, -0.000524924265515397], "logp": -0.0009469208418824715, "cache_id": 374, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.209342956542969, "actual_norm": 9.212175369262695, "relative_norm": 0.049939092879502045, "target_actual_norm": 9.21264362335205, "match_error": -0.00046825408935546875, "scalar": 9.209340139386768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-16.875421996576367, -3.529092249308976e-06], "logp": -16.875425525668618, "cache_id": 375, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.209342956542969, "actual_norm": 9.212175369262695, "relative_norm": 0.049939092879502045, "target_actual_norm": 9.21264362335205, "match_error": -0.00046825408935546875, "scalar": 9.209340139386768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.874478604826734, "first_log_odds": -16.875, "later_log_odds": 0.0005213951732656597, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-unrelated.npz", "array_sha256": "e4ff421b39652d614936d64b31fe4357284a7c7aad453bfdcdfe1371f93c42b7", "seconds": 0.32944708270952106, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456715264, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-ordinary", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.214137077331543, "actual_norm": 9.21259880065918, "relative_norm": 0.04994138829605563, "target_actual_norm": 9.21264362335205, "match_error": -4.482269287109375e-05, "scalar": 9.214137219825716, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 376, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0002815621989421054, -0.00044395291664634786], "logp": -0.0007255151155884532, "cache_id": 377, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.214137077331543, "actual_norm": 9.21259880065918, "relative_norm": 0.04994138829605563, "target_actual_norm": 9.21264362335205, "match_error": -4.482269287109375e-05, "scalar": 9.214137219825716, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.625281562198943, -4.952160633121937e-06], "logp": -17.625286514359576, "cache_id": 378, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.214137077331543, "actual_norm": 9.21259880065918, "relative_norm": 0.04994138829605563, "target_actual_norm": 9.21264362335205, "match_error": -4.482269287109375e-05, "scalar": 9.214137219825716, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.62456099924399, "first_log_odds": -17.625, "later_log_odds": 0.00043900075601044364, "first_differing_log_odds": -17.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-ordinary.npz", "array_sha256": "1e4acad4fd10d8c4cde0c4502331bd84ab9c0c4424008224738513d034990cf6", "seconds": 0.3356510838493705, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-random0", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.210431098937988, "actual_norm": 9.212693214416504, "relative_norm": 0.04994190011190879, "target_actual_norm": 9.21264362335205, "match_error": 4.9591064453125e-05, "scalar": 9.210429586226383, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 379, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0003467907689593248, -0.0005033144153944229], "logp": -0.0008501051843537476, "cache_id": 380, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.210431098937988, "actual_norm": 9.212693214416504, "relative_norm": 0.04994190011190879, "target_actual_norm": 9.21264362335205, "match_error": 4.9591064453125e-05, "scalar": 9.210429586226383, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.25034679076896, -2.4874574864556284e-06], "logp": -17.250349278226444, "cache_id": 381, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.210431098937988, "actual_norm": 9.212693214416504, "relative_norm": 0.04994190011190879, "target_actual_norm": 9.21264362335205, "match_error": 4.9591064453125e-05, "scalar": 9.210429586226383, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.24949917304209, "first_log_odds": -17.25, "later_log_odds": 0.0005008269579107605, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-random0.npz", "array_sha256": "cf88650037032834892f7c3826f94d75d07d8da82c386adf5a03dbd285fbc3c6", "seconds": 0.33723200019448996, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-random1", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.211554527282715, "actual_norm": 9.212611198425293, "relative_norm": 0.049941455504197964, "target_actual_norm": 9.21264362335205, "match_error": -3.24249267578125e-05, "scalar": 9.211554176512436, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 382, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0004052105001191334, -0.0004372149076800934], "logp": -0.0008424254077992268, "cache_id": 383, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.211554527282715, "actual_norm": 9.212611198425293, "relative_norm": 0.049941455504197964, "target_actual_norm": 9.21264362335205, "match_error": -3.24249267578125e-05, "scalar": 9.211554176512436, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.12540521050012, -3.036120432581697e-06], "logp": -17.12540824662055, "cache_id": 384, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.211554527282715, "actual_norm": 9.212611198425293, "relative_norm": 0.049941455504197964, "target_actual_norm": 9.21264362335205, "match_error": -3.24249267578125e-05, "scalar": 9.211554176512436, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.12456582121275, "first_log_odds": -17.125, "later_log_odds": 0.0004341787872483849, "first_differing_log_odds": -17.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-random1.npz", "array_sha256": "19cafe9a04a7ba664945914a6c340b0f9963403e64a8e155272a55574cc9d561", "seconds": 0.33769645914435387, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-random2", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216114044189453, "actual_norm": 9.212728500366211, "relative_norm": 0.04994209139662159, "target_actual_norm": 9.21264362335205, "match_error": 8.487701416015625e-05, "scalar": 9.21611403868792, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 385, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0003486601416426408, -0.0005260391882055582], "logp": -0.0008746993298481989, "cache_id": 386, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216114044189453, "actual_norm": 9.212728500366211, "relative_norm": 0.04994209139662159, "target_actual_norm": 9.21264362335205, "match_error": 8.487701416015625e-05, "scalar": 9.21611403868792, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-17.250348660141643, -3.7207062919144304e-06], "logp": -17.250352380847936, "cache_id": 387, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.4682159423828, "strength_one_norm": 9.443016052246094, "target_shrink_scale": 0.9767441616204054, "requested_norm": 9.216114044189453, "actual_norm": 9.212728500366211, "relative_norm": 0.04994209139662159, "target_actual_norm": 9.21264362335205, "match_error": 8.487701416015625e-05, "scalar": 9.21611403868792, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.249477681518087, "first_log_odds": -17.25, "later_log_odds": 0.0005223184819129756, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-random2.npz", "array_sha256": "0909d80e20e5ee767e2dc8235e6f6ca60d6ea6362aebe16f187139d6f1d3a968", "seconds": 0.3347650421783328, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456748032, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-main-bridge", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Mexico Pesos (MXN)<|im_end|>", "output_text": "Mexico Pesos (MXN)", "output_ids": [68831, 56856, 436, 320, 29423, 45, 8, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 220.57391357421875, 275.39208984375, 436.3020935058594, 56.22393035888672, 142.75096130371094, 33.14377975463867, 185.3197021484375, 7.628944396972656, 33.899749755859375, 65.20584106445312, 179.13137817382812, 184.2908935546875, 40.765098571777344, 134.34521484375, 73.5375747680664, 57.72054672241211, 88.7854995727539, 114.63944244384766, 77.43859100341797, 342.18121337890625, 324.25927734375, 139.7543182373047, 151.8772430419922, 500.4446716308594, 299.2565002441406, 371.14605712890625, 85.0601806640625, 762.2033081054688]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 220.57244873046875, 275.4890441894531, 436.2844543457031, 56.21160888671875, 142.76022338867188, 33.22768783569336, 185.30767822265625, 7.666599273681641, 33.956844329833984, 65.18629455566406, 179.1796417236328, 184.26991271972656, 40.827796936035156, 134.38873291015625, 73.53202819824219, 57.70545959472656, 88.79033660888672, 114.65921783447266, 77.45133972167969, 342.2469787597656, 324.20745849609375, 139.79135131835938, 151.9184112548828, 500.37274169921875, 299.2437744140625, 371.2080078125, 85.00548553466797, 762.1751708984375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.19369550049304962, 0.2623041272163391, 0.4467550218105316, 0.05832826718688011, 0.1387985199689865, 0.03138278052210808, 0.1819583922624588, 0.007485106587409973, 0.02980092167854309, 0.06395689398050308, 0.17223580181598663, 0.18867653608322144, 0.04210644215345383, 0.13566532731056213, 0.07069503515958786, 0.054682761430740356, 0.07146414369344711, 0.1142500787973404, 0.08614510297775269, 0.38022834062576294, 0.39096879959106445, 0.13934355974197388, 0.16024862229824066, 0.44117745757102966, 0.2882727086544037, 0.39733511209487915, 0.0817587673664093, 0.7163316011428833]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 388, "cached": true, "scoring": {"normalized": "Mexico Pesos (MXN)", "bare_format": false, "malformed": true, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Mexico Pesos (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Mexico", "start": 0, "end": 6}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-25.39655866740727, -0.0008695447277548751], "logp": -25.397428212135026, "cache_id": 389, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 220.57391357421875, 275.39208984375, 436.3020935058594, 56.22393035888672, 142.75096130371094, 33.14377975463867, 185.3197021484375, 7.628944396972656, 33.899749755859375, 65.20584106445312, 179.13137817382812, 184.2908935546875, 40.765098571777344, 134.34521484375, 73.5375747680664, 57.72054672241211, 88.7854995727539, 114.63944244384766, 77.43859100341797, 342.18121337890625, 324.25927734375, 139.7543182373047, 151.8772430419922, 500.4446716308594, 299.2565002441406, 371.14605712890625, 85.0601806640625, 762.2033081054688]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 220.57244873046875, 275.4890441894531, 436.2844543457031, 56.21160888671875, 142.76022338867188, 33.22768783569336, 185.30767822265625, 7.666599273681641, 33.956844329833984, 65.18629455566406, 179.1796417236328, 184.26991271972656, 40.827796936035156, 134.38873291015625, 73.53202819824219, 57.70545959472656, 88.79033660888672, 114.65921783447266, 77.45133972167969, 342.2469787597656, 324.20745849609375, 139.79135131835938, 151.9184112548828, 500.37274169921875, 299.2437744140625, 371.2080078125, 85.00548553466797, 762.1751708984375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.19369550049304962, 0.2623041272163391, 0.4467550218105316, 0.05832826718688011, 0.1387985199689865, 0.03138278052210808, 0.1819583922624588, 0.007485106587409973, 0.02980092167854309, 0.06395689398050308, 0.17223580181598663, 0.18867653608322144, 0.04210644215345383, 0.13566532731056213, 0.07069503515958786, 0.054682761430740356, 0.07146414369344711, 0.1142500787973404, 0.08614510297775269, 0.38022834062576294, 0.39096879959106445, 0.13934355974197388, 0.16024862229824066, 0.44117745757102966, 0.2882727086544037, 0.39733511209487915, 0.0817587673664093, 0.7163316011428833]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.8965586674072699, -7.795082072458534e-06], "logp": -0.8965664624893424, "cache_id": 390, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 220.57391357421875, 275.39208984375, 436.3020935058594, 56.22393035888672, 142.75096130371094, 33.14377975463867, 185.3197021484375, 7.628944396972656, 33.899749755859375, 65.20584106445312, 179.13137817382812, 184.2908935546875, 40.765098571777344, 134.34521484375, 73.5375747680664, 57.72054672241211, 88.7854995727539, 114.63944244384766, 77.43859100341797, 342.18121337890625, 324.25927734375, 139.7543182373047, 151.8772430419922, 500.4446716308594, 299.2565002441406, 371.14605712890625, 85.0601806640625, 762.2033081054688]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 220.57244873046875, 275.4890441894531, 436.2844543457031, 56.21160888671875, 142.76022338867188, 33.22768783569336, 185.30767822265625, 7.666599273681641, 33.956844329833984, 65.18629455566406, 179.1796417236328, 184.26991271972656, 40.827796936035156, 134.38873291015625, 73.53202819824219, 57.70545959472656, 88.79033660888672, 114.65921783447266, 77.45133972167969, 342.2469787597656, 324.20745849609375, 139.79135131835938, 151.9184112548828, 500.37274169921875, 299.2437744140625, 371.2080078125, 85.00548553466797, 762.1751708984375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.19369550049304962, 0.2623041272163391, 0.4467550218105316, 0.05832826718688011, 0.1387985199689865, 0.03138278052210808, 0.1819583922624588, 0.007485106587409973, 0.02980092167854309, 0.06395689398050308, 0.17223580181598663, 0.18867653608322144, 0.04210644215345383, 0.13566532731056213, 0.07069503515958786, 0.054682761430740356, 0.07146414369344711, 0.1142500787973404, 0.08614510297775269, 0.38022834062576294, 0.39096879959106445, 0.13934355974197388, 0.16024862229824066, 0.44117745757102966, 0.2882727086544037, 0.39733511209487915, 0.0817587673664093, 0.7163316011428833]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 24.500861749645683, "first_log_odds": 24.5, "later_log_odds": 0.0008617496456828633, "first_differing_log_odds": 24.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-main-bridge.npz", "array_sha256": "ae6aee06469506e7bf559fa0c994aef2ccafee0dcfb7ae5113844d7d7917e23a", "seconds": 0.5203876248560846, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-baseline", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 391, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.625020364890393, -0.0009002671816298048], "logp": -12.625920632072022, "cache_id": 392, "trace": []}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.250020364890393, -2.5841714787175807e-05], "logp": -22.25004620660518, "cache_id": 393, "trace": []}, "log_odds": -9.624125574533158, "first_log_odds": -9.625, "later_log_odds": 0.0008744254668417994, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-baseline.npz", "array_sha256": "7c1d1c88669887e0db4afb503e87269979811423b4cde18e84ef46ded39578a5", "seconds": 1.3107978329062462, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-noop", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 6.475711345672607, "match_error": -6.475711345672607, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 394, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.625020364890393, -0.0009002671816298048], "logp": -12.625920632072022, "cache_id": 395, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 6.475711345672607, "match_error": -6.475711345672607, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.250020364890393, -2.5841714787175807e-05], "logp": -22.25004620660518, "cache_id": 396, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 6.475711345672607, "match_error": -6.475711345672607, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.624125574533158, "first_log_odds": -9.625, "later_log_odds": 0.0008744254668417994, "first_differing_log_odds": -9.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-noop.npz", "array_sha256": "bef9de1eedea7fed328e03af9a4cad21f155e6b2204951a1a33df0483480a2bc", "seconds": 1.2630524588748813, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-j", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473813056945801, "actual_norm": 6.475711345672607, "relative_norm": 0.03489556781573092, "target_actual_norm": 6.475711345672607, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 397, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-13.750008738970186, -0.0010163936186351082], "logp": -13.75102513258882, "cache_id": 398, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473813056945801, "actual_norm": 6.475711345672607, "relative_norm": 0.03489556781573092, "target_actual_norm": 6.475711345672607, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.125008738970187, -3.1673281502413454e-05], "logp": -22.12504041225169, "cache_id": 399, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473813056945801, "actual_norm": 6.475711345672607, "relative_norm": 0.03489556781573092, "target_actual_norm": 6.475711345672607, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.374015279662869, "first_log_odds": -8.375000000000002, "later_log_odds": 0.0009847203371329272, "first_differing_log_odds": -8.375000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-j.npz", "array_sha256": "cf22f5715ada0546b7bb3c3b35f6344054a63004ce0f035f299b17678d5e26fc", "seconds": 1.2622897499240935, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-opposite", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.476896286010742, "actual_norm": 6.474829196929932, "relative_norm": 0.03489081419413934, "target_actual_norm": 6.475711345672607, "match_error": -0.0008821487426757812, "scalar": 6.4768970838340465, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 400, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.875016403320114, -0.0008058635402490462], "logp": -12.875822266860363, "cache_id": 401, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.476896286010742, "actual_norm": 6.474829196929932, "relative_norm": 0.03489081419413934, "target_actual_norm": 6.475711345672607, "match_error": -0.0008821487426757812, "scalar": 6.4768970838340465, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.750016403320114, -2.8143349688799044e-05], "logp": -22.750044546669802, "cache_id": 402, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.476896286010742, "actual_norm": 6.474829196929932, "relative_norm": 0.03489081419413934, "target_actual_norm": 6.475711345672607, "match_error": -0.0008821487426757812, "scalar": 6.4768970838340465, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.87422227980944, "first_log_odds": -9.875, "later_log_odds": 0.0007777201905607711, "first_differing_log_odds": -9.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-opposite.npz", "array_sha256": "549f5d4094033fd6cbc2b0212280bcb58e90a1bbd925880178137d83b888f17f", "seconds": 1.2480522082187235, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-unrelated", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.467709064483643, "actual_norm": 6.475856781005859, "relative_norm": 0.034896351520912236, "target_actual_norm": 6.475711345672607, "match_error": 0.00014543533325195312, "scalar": 6.467707613082894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 403, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.750016095027293, -0.0008978058270655926], "logp": -12.750913900854359, "cache_id": 404, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.467709064483643, "actual_norm": 6.475856781005859, "relative_norm": 0.034896351520912236, "target_actual_norm": 6.475711345672607, "match_error": 0.00014543533325195312, "scalar": 6.467707613082894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.375016095027295, -2.8674614434683105e-05], "logp": -22.37504476964173, "cache_id": 405, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.467709064483643, "actual_norm": 6.475856781005859, "relative_norm": 0.034896351520912236, "target_actual_norm": 6.475711345672607, "match_error": 0.00014543533325195312, "scalar": 6.467707613082894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.62413086878737, "first_log_odds": -9.625000000000002, "later_log_odds": 0.0008691312126316575, "first_differing_log_odds": -9.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-unrelated.npz", "array_sha256": "5e6369b6a2bc4fd14476662522fb55bfae372ae6135df6209ced736557034ed8", "seconds": 1.2654635421931744, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-ordinary", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473043441772461, "actual_norm": 6.475686550140381, "relative_norm": 0.03489543420042132, "target_actual_norm": 6.475711345672607, "match_error": -2.47955322265625e-05, "scalar": 6.4730434348093695, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 406, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.875017254996868, -0.000824990666546854], "logp": -12.875842245663415, "cache_id": 407, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473043441772461, "actual_norm": 6.475686550140381, "relative_norm": 0.03489543420042132, "target_actual_norm": 6.475711345672607, "match_error": -2.47955322265625e-05, "scalar": 6.4730434348093695, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.375017254996866, -2.7588788266261798e-05], "logp": -22.375044843785133, "cache_id": 408, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.473043441772461, "actual_norm": 6.475686550140381, "relative_norm": 0.03489543420042132, "target_actual_norm": 6.475711345672607, "match_error": -2.47955322265625e-05, "scalar": 6.4730434348093695, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.499202598121718, "first_log_odds": -9.499999999999998, "later_log_odds": 0.0007974018782803682, "first_differing_log_odds": -9.499999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-ordinary.npz", "array_sha256": "cc85d3bfe35ed3704761478c040ccd36ea10252a185c558cecc67d9d1af46929", "seconds": 1.2612790423445404, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-random0", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.46919059753418, "actual_norm": 6.475771427154541, "relative_norm": 0.034895891575904184, "target_actual_norm": 6.475711345672607, "match_error": 6.008148193359375e-05, "scalar": 6.4691897857846925, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 409, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-13.625010029034813, -0.0009348342969127946], "logp": -13.625944863331727, "cache_id": 410, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.46919059753418, "actual_norm": 6.475771427154541, "relative_norm": 0.034895891575904184, "target_actual_norm": 6.475711345672607, "match_error": 6.008148193359375e-05, "scalar": 6.4691897857846925, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.875010029034815, -3.785809621391756e-05], "logp": -22.87504788713103, "cache_id": 411, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.46919059753418, "actual_norm": 6.475771427154541, "relative_norm": 0.034895891575904184, "target_actual_norm": 6.475711345672607, "match_error": 6.008148193359375e-05, "scalar": 6.4691897857846925, "random_seed": 20271608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.249103023799302, "first_log_odds": -9.250000000000002, "later_log_odds": 0.0008969762006998394, "first_differing_log_odds": -9.250000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-random0.npz", "array_sha256": "a9f712f0a8fe1f1d91f00cec0336398fb1064a2cb8cc91e540cc9d4e96e45e16", "seconds": 1.2413381249643862, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-random1", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.470753192901611, "actual_norm": 6.476206302642822, "relative_norm": 0.034898234982872575, "target_actual_norm": 6.475711345672607, "match_error": 0.0004949569702148438, "scalar": 6.470754172753544, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 412, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-12.625020556794565, -0.0009936280197064425], "logp": -12.626014184814272, "cache_id": 413, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.470753192901611, "actual_norm": 6.476206302642822, "relative_norm": 0.034898234982872575, "target_actual_norm": 6.475711345672607, "match_error": 0.0004949569702148438, "scalar": 6.470754172753544, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.250020556794567, -3.775291563688604e-05], "logp": -22.250058309710205, "cache_id": 414, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.470753192901611, "actual_norm": 6.476206302642822, "relative_norm": 0.034898234982872575, "target_actual_norm": 6.475711345672607, "match_error": 0.0004949569702148438, "scalar": 6.470754172753544, "random_seed": 20271609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.624044124895933, "first_log_odds": -9.625000000000002, "later_log_odds": 0.000955875104068582, "first_differing_log_odds": -9.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-random1.npz", "array_sha256": "99fab518eb5d3d21346dbb7c581f23dd47906e4c22203e493dad76195f69abce", "seconds": 1.2630743752233684, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-random2", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.478255748748779, "actual_norm": 6.475471496582031, "relative_norm": 0.03489427534456305, "target_actual_norm": 6.475711345672607, "match_error": -0.00023984909057617188, "scalar": 6.478255742144029, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 415, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-13.250012707713239, -0.0008290048319672629], "logp": -13.250841712545206, "cache_id": 416, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.478255748748779, "actual_norm": 6.475471496582031, "relative_norm": 0.03489427534456305, "target_actual_norm": 6.475711345672607, "match_error": -0.00023984909057617188, "scalar": 6.478255742144029, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.50001270771324, -2.6451509170033723e-05], "logp": -22.50003915922241, "cache_id": 417, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.57403564453125, "strength_one_norm": 6.473813056945801, "target_shrink_scale": 1.0, "requested_norm": 6.478255748748779, "actual_norm": 6.475471496582031, "relative_norm": 0.03489427534456305, "target_actual_norm": 6.475711345672607, "match_error": -0.00023984909057617188, "scalar": 6.478255742144029, "random_seed": 20271610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.249197446677204, "first_log_odds": -9.25, "later_log_odds": 0.0008025533227957737, "first_differing_log_odds": -9.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-random2.npz", "array_sha256": "9deabf23c08741f064967e1c622a37ef5cb3cc76020667b3a61440aff5e95c35", "seconds": 1.266152665950358, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-07-paraphrase-bridge", "case_id": "eval-07", "city": "Rio de Janeiro", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 26961, 409, 60323, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 208.81910705566406, 235.346923828125, 456.82464599609375, 34.024871826171875, 74.32443237304688, 39.93043899536133, 157.19244384765625, 9.961699485778809, 39.96751403808594, 62.64663314819336, 193.73512268066406, 199.45803833007812, 49.31147384643555, 106.49295043945312, 101.90519714355469, 51.86021041870117, 335.161376953125, 440.63885498046875, 175.744384765625, 152.09645080566406, 525.437255859375, 296.4798889160156, 384.0897521972656, 86.43350982666016, 385.1339416503906]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 208.80062866210938, 235.36285400390625, 456.77386474609375, 34.06959915161133, 74.23717498779297, 39.95375061035156, 157.15921020507812, 10.021615982055664, 40.01506805419922, 62.74210739135742, 193.7491912841797, 199.48570251464844, 49.34867858886719, 106.4876937866211, 101.95777130126953, 51.80979919433594, 335.1032409667969, 440.6718444824219, 175.77020263671875, 152.15452575683594, 525.5482177734375, 296.4859313964844, 383.9911193847656, 86.37113952636719, 385.2076416015625]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.1813650280237198, 0.22476032376289368, 0.4642733931541443, 0.03352281451225281, 0.06769181787967682, 0.03746892884373665, 0.14525990188121796, 0.009785773232579231, 0.03438742086291313, 0.05981782451272011, 0.18341584503650665, 0.2026492953300476, 0.05051146447658539, 0.10494957119226456, 0.09577454626560211, 0.0539124421775341, 0.3788018524646759, 0.5152119398117065, 0.17129436135292053, 0.16174279153347015, 0.4564167857170105, 0.2797546684741974, 0.40505608916282654, 0.08296285569667816, 0.4035313129425049]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 418, "cached": true, "scoring": {"normalized": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Rio de Janeiro is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 43, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-29.12503738134312, -0.0015911365665372621], "logp": -29.12662851790966, "cache_id": 419, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 208.81910705566406, 235.346923828125, 456.82464599609375, 34.024871826171875, 74.32443237304688, 39.93043899536133, 157.19244384765625, 9.961699485778809, 39.96751403808594, 62.64663314819336, 193.73512268066406, 199.45803833007812, 49.31147384643555, 106.49295043945312, 101.90519714355469, 51.86021041870117, 335.161376953125, 440.63885498046875, 175.744384765625, 152.09645080566406, 525.437255859375, 296.4798889160156, 384.0897521972656, 86.43350982666016, 385.1339416503906]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 208.80062866210938, 235.36285400390625, 456.77386474609375, 34.06959915161133, 74.23717498779297, 39.95375061035156, 157.15921020507812, 10.021615982055664, 40.01506805419922, 62.74210739135742, 193.7491912841797, 199.48570251464844, 49.34867858886719, 106.4876937866211, 101.95777130126953, 51.80979919433594, 335.1032409667969, 440.6718444824219, 175.77020263671875, 152.15452575683594, 525.5482177734375, 296.4859313964844, 383.9911193847656, 86.37113952636719, 385.2076416015625]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.1813650280237198, 0.22476032376289368, 0.4642733931541443, 0.03352281451225281, 0.06769181787967682, 0.03746892884373665, 0.14525990188121796, 0.009785773232579231, 0.03438742086291313, 0.05981782451272011, 0.18341584503650665, 0.2026492953300476, 0.05051146447658539, 0.10494957119226456, 0.09577454626560211, 0.0539124421775341, 0.3788018524646759, 0.5152119398117065, 0.17129436135292053, 0.16174279153347015, 0.4564167857170105, 0.2797546684741974, 0.40505608916282654, 0.08296285569667816, 0.4035313129425049]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-14.12503738134312, -6.621678202256459e-05], "logp": -14.125103598125142, "cache_id": 420, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 208.81910705566406, 235.346923828125, 456.82464599609375, 34.024871826171875, 74.32443237304688, 39.93043899536133, 157.19244384765625, 9.961699485778809, 39.96751403808594, 62.64663314819336, 193.73512268066406, 199.45803833007812, 49.31147384643555, 106.49295043945312, 101.90519714355469, 51.86021041870117, 335.161376953125, 440.63885498046875, 175.744384765625, 152.09645080566406, 525.437255859375, 296.4798889160156, 384.0897521972656, 86.43350982666016, 385.1339416503906]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 208.80062866210938, 235.36285400390625, 456.77386474609375, 34.06959915161133, 74.23717498779297, 39.95375061035156, 157.15921020507812, 10.021615982055664, 40.01506805419922, 62.74210739135742, 193.7491912841797, 199.48570251464844, 49.34867858886719, 106.4876937866211, 101.95777130126953, 51.80979919433594, 335.1032409667969, 440.6718444824219, 175.77020263671875, 152.15452575683594, 525.5482177734375, 296.4859313964844, 383.9911193847656, 86.37113952636719, 385.2076416015625]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.1813650280237198, 0.22476032376289368, 0.4642733931541443, 0.03352281451225281, 0.06769181787967682, 0.03746892884373665, 0.14525990188121796, 0.009785773232579231, 0.03438742086291313, 0.05981782451272011, 0.18341584503650665, 0.2026492953300476, 0.05051146447658539, 0.10494957119226456, 0.09577454626560211, 0.0539124421775341, 0.3788018524646759, 0.5152119398117065, 0.17129436135292053, 0.16174279153347015, 0.4564167857170105, 0.2797546684741974, 0.40505608916282654, 0.08296285569667816, 0.4035313129425049]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.001524919784517, "first_log_odds": 15.0, "later_log_odds": 0.0015249197845168538, "first_differing_log_odds": 15.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-07-paraphrase-bridge.npz", "array_sha256": "e96d0b6e0b6e8a3ca1862f633679fb39158394803f488f0dffd4a86048b6163e", "seconds": 1.2614471670240164, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-baseline", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 421, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-9.230885671156658e-05, -0.009571586641829161], "logp": -0.009663895498540728, "cache_id": 422, "trace": []}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.37509230885671, -2.8350668086281364e-06], "logp": -21.37509514392352, "cache_id": 423, "trace": []}, "log_odds": -21.36543124842498, "first_log_odds": -21.375, "later_log_odds": 0.009568751575020684, "first_differing_log_odds": -21.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-baseline.npz", "array_sha256": "511f66b1cfced566b78c0a967959fc45d483dfb10c0df42c9ca5461150e5ec0c", "seconds": 0.39243070827797055, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-noop", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 4.954415321350098, "match_error": -4.954415321350098, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 424, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-9.230885671156658e-05, -0.009571586641829161], "logp": -0.009663895498540728, "cache_id": 425, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 4.954415321350098, "match_error": -4.954415321350098, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.37509230885671, -2.8350668086281364e-06], "logp": -21.37509514392352, "cache_id": 426, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 4.954415321350098, "match_error": -4.954415321350098, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.36543124842498, "first_log_odds": -21.375, "later_log_odds": 0.009568751575020684, "first_differing_log_odds": -21.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-noop.npz", "array_sha256": "e0633f00d9429a2c4a5e98c148f2913e70201f5ce4ed26e0e69ce9a0d894a811", "seconds": 0.3392655001953244, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-j", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.938234329223633, "actual_norm": 4.954415321350098, "relative_norm": 0.026249243407040667, "target_actual_norm": 4.954415321350098, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 427, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00015419046790197296, -0.010953432650443799], "logp": -0.011107623118345771, "cache_id": 428, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.938234329223633, "actual_norm": 4.954415321350098, "relative_norm": 0.026249243407040667, "target_actual_norm": 4.954415321350098, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.500154190467903, -3.269640075798944e-06], "logp": -20.500157460107978, "cache_id": 429, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.938234329223633, "actual_norm": 4.954415321350098, "relative_norm": 0.026249243407040667, "target_actual_norm": 4.954415321350098, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.489049836989633, "first_log_odds": -20.5, "later_log_odds": 0.01095016301036722, "first_differing_log_odds": -20.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-j.npz", "array_sha256": "d45fb2e7f903250aa41c0b6607c99efe980c2576d2aa455a0b00be25d24eec62", "seconds": 0.34174845833331347, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-opposite", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951938629150391, "actual_norm": 4.954466342926025, "relative_norm": 0.02624951372708424, "target_actual_norm": 4.954415321350098, "match_error": 5.1021575927734375e-05, "scalar": 4.951939474460232, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 430, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.0001172379698770404, -0.008322701758475938], "logp": -0.008439939728352979, "cache_id": 431, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951938629150391, "actual_norm": 4.954466342926025, "relative_norm": 0.02624951372708424, "target_actual_norm": 4.954415321350098, "match_error": 5.1021575927734375e-05, "scalar": 4.951939474460232, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.375117237969878, -2.969076302690731e-06], "logp": -21.37512020704618, "cache_id": 432, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951938629150391, "actual_norm": 4.954466342926025, "relative_norm": 0.02624951372708424, "target_actual_norm": 4.954415321350098, "match_error": 5.1021575927734375e-05, "scalar": 4.951939474460232, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.36668026731783, "first_log_odds": -21.375, "later_log_odds": 0.008319732682171121, "first_differing_log_odds": -21.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-opposite.npz", "array_sha256": "db3ef60a648e9c690877ab68f2d1a72af2753c64bafb6a062503e818ce746c0f", "seconds": 0.33951033325865865, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-unrelated", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.952770233154297, "actual_norm": 4.9550628662109375, "relative_norm": 0.02625267419787403, "target_actual_norm": 4.954415321350098, "match_error": 0.0006475448608398438, "scalar": 4.952771056621714, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 433, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00011979966399116002, -0.009558737773365716], "logp": -0.009678537437356876, "cache_id": 434, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.952770233154297, "actual_norm": 4.9550628662109375, "relative_norm": 0.02625267419787403, "target_actual_norm": 4.954415321350098, "match_error": 0.0006475448608398438, "scalar": 4.952771056621714, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.12511979966399, -2.9432614863358482e-06], "logp": -21.125122742925477, "cache_id": 435, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.952770233154297, "actual_norm": 4.9550628662109375, "relative_norm": 0.02625267419787403, "target_actual_norm": 4.954415321350098, "match_error": 0.0006475448608398438, "scalar": 4.952771056621714, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.11544420548812, "first_log_odds": -21.125, "later_log_odds": 0.009555794511879157, "first_differing_log_odds": -21.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-unrelated.npz", "array_sha256": "a37f1e10613c5c5e4031ec8bd9f103288fb79cc5328a03a013f3ca6349a4e59a", "seconds": 0.33266466623172164, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-ordinary", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.943756580352783, "actual_norm": 4.954338073730469, "relative_norm": 0.026248834137441987, "target_actual_norm": 4.954415321350098, "match_error": -7.724761962890625e-05, "scalar": 4.943755950007471, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 436, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00011298718276650752, -0.010672555599771798], "logp": -0.010785542782538305, "cache_id": 437, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.943756580352783, "actual_norm": 4.954338073730469, "relative_norm": 0.026248834137441987, "target_actual_norm": 4.954415321350098, "match_error": -7.724761962890625e-05, "scalar": 4.943755950007471, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.125112987182767, -2.5358785292492857e-06], "logp": -21.125115523061297, "cache_id": 438, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.943756580352783, "actual_norm": 4.954338073730469, "relative_norm": 0.026248834137441987, "target_actual_norm": 4.954415321350098, "match_error": -7.724761962890625e-05, "scalar": 4.943755950007471, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.114329980278757, "first_log_odds": -21.125, "later_log_odds": 0.010670019721242596, "first_differing_log_odds": -21.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-ordinary.npz", "array_sha256": "5af7978bc18114b4275c7985336ecc4fe37f2c60fa269e93c2f8b41b34dc9566", "seconds": 0.3394056670367718, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-random0", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.95256233215332, "actual_norm": 4.954476833343506, "relative_norm": 0.02624956930690628, "target_actual_norm": 4.954415321350098, "match_error": 6.151199340820312e-05, "scalar": 4.952563161081343, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 439, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00011417490216180119, -0.009579433474231881], "logp": -0.009693608376393683, "cache_id": 440, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.95256233215332, "actual_norm": 4.954476833343506, "relative_norm": 0.02624956930690628, "target_actual_norm": 4.954415321350098, "match_error": 6.151199340820312e-05, "scalar": 4.952563161081343, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.125114174902162, -4.683275444348759e-06], "logp": -21.125118858177608, "cache_id": 441, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.95256233215332, "actual_norm": 4.954476833343506, "relative_norm": 0.02624956930690628, "target_actual_norm": 4.954415321350098, "match_error": 6.151199340820312e-05, "scalar": 4.952563161081343, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.115425249801213, "first_log_odds": -21.125, "later_log_odds": 0.009574750198787285, "first_differing_log_odds": -21.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-random0.npz", "array_sha256": "438d1f61e970ac946eb90af2d4fb50d7df23ef8904c26bef67bc54eac0ec0111", "seconds": 0.33894041599705815, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-random1", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.9490437507629395, "actual_norm": 4.954279899597168, "relative_norm": 0.026248525922065207, "target_actual_norm": 4.954415321350098, "match_error": -0.0001354217529296875, "scalar": 4.949043111590981, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 442, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00011352034826353574, -0.009477876702450788], "logp": -0.009591397050714323, "cache_id": 443, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.9490437507629395, "actual_norm": 4.954279899597168, "relative_norm": 0.026248525922065207, "target_actual_norm": 4.954415321350098, "match_error": -0.0001354217529296875, "scalar": 4.949043111590981, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.000113520348265, -3.488491462705718e-06], "logp": -21.000117008839727, "cache_id": 444, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.9490437507629395, "actual_norm": 4.954279899597168, "relative_norm": 0.026248525922065207, "target_actual_norm": 4.954415321350098, "match_error": -0.0001354217529296875, "scalar": 4.949043111590981, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.990525611789014, "first_log_odds": -21.0, "later_log_odds": 0.009474388210986007, "first_differing_log_odds": -21.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-random1.npz", "array_sha256": "7828850e48c53fe96bb664586fc23d7f7b6e547733c186479112c041d40d6399", "seconds": 0.33821841701865196, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-random2", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "BRL<|im_end|>", "output_text": "BRL", "output_ids": [33, 4728, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951213359832764, "actual_norm": 4.95465087890625, "relative_norm": 0.02625049142668108, "target_actual_norm": 4.954415321350098, "match_error": 0.00023555755615234375, "scalar": 4.951214202518258, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 445, "cached": true, "scoring": {"normalized": "BRL", "bare_format": true, "malformed": false, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-0.00012201709171343751, -0.00963994657320993], "logp": -0.009761963664923368, "cache_id": 446, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951213359832764, "actual_norm": 4.95465087890625, "relative_norm": 0.02625049142668108, "target_actual_norm": 4.954415321350098, "match_error": 0.00023555755615234375, "scalar": 4.951214202518258, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-21.250122017091712, -4.5018882656642874e-06], "logp": -21.25012651897998, "cache_id": 447, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7450714111328, "strength_one_norm": 4.938234329223633, "target_shrink_scale": 1.0, "requested_norm": 4.951213359832764, "actual_norm": 4.95465087890625, "relative_norm": 0.02625049142668108, "target_actual_norm": 4.954415321350098, "match_error": 0.00023555755615234375, "scalar": 4.951214202518258, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.240364555315054, "first_log_odds": -21.25, "later_log_odds": 0.009635444684946037, "first_differing_log_odds": -21.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-random2.npz", "array_sha256": "efdb5e1ea7e9352f196cad01f04c656986ffd3058556eca61d4789f29874973f", "seconds": 0.338100790977478, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-main-bridge", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Mexico Pesos (MXN)<|im_end|>", "output_text": "Mexico Pesos (MXN)", "output_ids": [68831, 56856, 436, 320, 29423, 45, 8, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 389.3558044433594, 329.46502685546875, 25.110485076904297, 147.63247680664062, 19.847015380859375, 183.0570526123047, 4.916019916534424, 28.986358642578125, 67.32933807373047, 185.48410034179688, 185.64036560058594, 42.97850799560547, 114.97801971435547, 74.97037506103516, 58.084041595458984, 92.68914031982422, 125.782958984375, 72.55477142333984, 341.7226867675781, 331.5430603027344, 151.11257934570312, 146.188720703125, 519.692138671875, 307.72833251953125, 372.6492004394531, 75.57746887207031, 781.5271606445312]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 389.37103271484375, 329.4588623046875, 25.135744094848633, 147.6653594970703, 19.88009262084961, 183.09397888183594, 5.121289253234863, 28.987703323364258, 67.36868286132812, 185.56884765625, 185.64852905273438, 42.998374938964844, 114.99250030517578, 74.9764175415039, 58.09710693359375, 92.6756591796875, 125.79936218261719, 72.55784606933594, 341.80523681640625, 331.4996337890625, 151.13502502441406, 146.14967346191406, 519.7396240234375, 307.7445373535156, 372.5794677734375, 75.61312866210938, 781.5531005859375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.31923308968544006, 0.3382895886898041, 0.026714108884334564, 0.1486298143863678, 0.01857737824320793, 0.1789248287677765, 0.005012577399611473, 0.025555821135640144, 0.06612910330295563, 0.17833594977855682, 0.19051200151443481, 0.044547051191329956, 0.11743675917387009, 0.07214801758527756, 0.05488748103380203, 0.0745638906955719, 0.12559440732002258, 0.08127332478761673, 0.3784286081790924, 0.3996467590332031, 0.15012802183628082, 0.1541382372379303, 0.4548617899417877, 0.29779350757598877, 0.40045294165611267, 0.0729840025305748, 0.7384077906608582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 448, "cached": true, "scoring": {"normalized": "Mexico Pesos (MXN)", "bare_format": false, "malformed": true, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Mexico Pesos (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Mexico", "start": 0, "end": 6}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-24.19181438896523, -0.008955479131140425], "logp": -24.200769868096373, "cache_id": 449, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 389.3558044433594, 329.46502685546875, 25.110485076904297, 147.63247680664062, 19.847015380859375, 183.0570526123047, 4.916019916534424, 28.986358642578125, 67.32933807373047, 185.48410034179688, 185.64036560058594, 42.97850799560547, 114.97801971435547, 74.97037506103516, 58.084041595458984, 92.68914031982422, 125.782958984375, 72.55477142333984, 341.7226867675781, 331.5430603027344, 151.11257934570312, 146.188720703125, 519.692138671875, 307.72833251953125, 372.6492004394531, 75.57746887207031, 781.5271606445312]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 389.37103271484375, 329.4588623046875, 25.135744094848633, 147.6653594970703, 19.88009262084961, 183.09397888183594, 5.121289253234863, 28.987703323364258, 67.36868286132812, 185.56884765625, 185.64852905273438, 42.998374938964844, 114.99250030517578, 74.9764175415039, 58.09710693359375, 92.6756591796875, 125.79936218261719, 72.55784606933594, 341.80523681640625, 331.4996337890625, 151.13502502441406, 146.14967346191406, 519.7396240234375, 307.7445373535156, 372.5794677734375, 75.61312866210938, 781.5531005859375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.31923308968544006, 0.3382895886898041, 0.026714108884334564, 0.1486298143863678, 0.01857737824320793, 0.1789248287677765, 0.005012577399611473, 0.025555821135640144, 0.06612910330295563, 0.17833594977855682, 0.19051200151443481, 0.044547051191329956, 0.11743675917387009, 0.07214801758527756, 0.05488748103380203, 0.0745638906955719, 0.12559440732002258, 0.08127332478761673, 0.3784286081790924, 0.3996467590332031, 0.15012802183628082, 0.1541382372379303, 0.4548617899417877, 0.29779350757598877, 0.40045294165611267, 0.0729840025305748, 0.7384077906608582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-2.1918143889652297, -7.474186906350036e-06], "logp": -2.191821863152136, "cache_id": 450, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 2.035207986831665, 11.395103454589844, 124.14926147460938, 31.77028465270996, 15.66701889038086, 389.3558044433594, 329.46502685546875, 25.110485076904297, 147.63247680664062, 19.847015380859375, 183.0570526123047, 4.916019916534424, 28.986358642578125, 67.32933807373047, 185.48410034179688, 185.64036560058594, 42.97850799560547, 114.97801971435547, 74.97037506103516, 58.084041595458984, 92.68914031982422, 125.782958984375, 72.55477142333984, 341.7226867675781, 331.5430603027344, 151.11257934570312, 146.188720703125, 519.692138671875, 307.72833251953125, 372.6492004394531, 75.57746887207031, 781.5271606445312]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 2.092207193374634, 11.457947731018066, 124.11993408203125, 31.825551986694336, 15.66738224029541, 389.37103271484375, 329.4588623046875, 25.135744094848633, 147.6653594970703, 19.88009262084961, 183.09397888183594, 5.121289253234863, 28.987703323364258, 67.36868286132812, 185.56884765625, 185.64852905273438, 42.998374938964844, 114.99250030517578, 74.9764175415039, 58.09710693359375, 92.6756591796875, 125.79936218261719, 72.55784606933594, 341.80523681640625, 331.4996337890625, 151.13502502441406, 146.14967346191406, 519.7396240234375, 307.7445373535156, 372.5794677734375, 75.61312866210938, 781.5531005859375]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.0019205727148801088, 0.010052989237010479, 0.10097815096378326, 0.02724013477563858, 0.013806109316647053, 0.31923308968544006, 0.3382895886898041, 0.026714108884334564, 0.1486298143863678, 0.01857737824320793, 0.1789248287677765, 0.005012577399611473, 0.025555821135640144, 0.06612910330295563, 0.17833594977855682, 0.19051200151443481, 0.044547051191329956, 0.11743675917387009, 0.07214801758527756, 0.05488748103380203, 0.0745638906955719, 0.12559440732002258, 0.08127332478761673, 0.3784286081790924, 0.3996467590332031, 0.15012802183628082, 0.1541382372379303, 0.4548617899417877, 0.29779350757598877, 0.40045294165611267, 0.0729840025305748, 0.7384077906608582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 22.008948004944237, "first_log_odds": 22.0, "later_log_odds": 0.008948004944237198, "first_differing_log_odds": 22.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-main-bridge.npz", "array_sha256": "aaa5d108e652e9a665b56945637dcb2f180470634a60f1bc67ff334c408f7a52", "seconds": 0.514764416962862, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-baseline", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 451, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.288983873492207, -0.1647327474952803], "logp": -3.453716620987487, "cache_id": 452, "trace": []}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.038983873492207, -8.92840335756017e-06], "logp": -20.038992801895564, "cache_id": 453, "trace": []}, "log_odds": -16.585276180908075, "first_log_odds": -16.75, "later_log_odds": 0.16472381909192535, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-baseline.npz", "array_sha256": "9295085943b3235b502403799c647805faf2207adeb86767ef0802b50f7e0b0f", "seconds": 1.2076108753681183, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-noop", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 3.6857073307037354, "match_error": -3.6857073307037354, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 454, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.288983873492207, -0.1647327474952803], "logp": -3.453716620987487, "cache_id": 455, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 3.6857073307037354, "match_error": -3.6857073307037354, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.038983873492207, -8.92840335756017e-06], "logp": -20.038992801895564, "cache_id": 456, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 3.6857073307037354, "match_error": -3.6857073307037354, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.585276180908075, "first_log_odds": -16.75, "later_log_odds": 0.16472381909192535, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-noop.npz", "array_sha256": "d14bf0757e2cd59af31e42bffcac211f63b56ed4fdaaabcb766a1d2faf708f49", "seconds": 1.180358957964927, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-j", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6679317951202393, "actual_norm": 3.6857073307037354, "relative_norm": 0.019386628533194793, "target_actual_norm": 3.6857073307037354, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 457, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.050045445059368, -0.20697549353083206], "logp": -3.2570209385902, "cache_id": 458, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6679317951202393, "actual_norm": 3.6857073307037354, "relative_norm": 0.019386628533194793, "target_actual_norm": 3.6857073307037354, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-19.550045445059368, -8.571129113062633e-06], "logp": -19.55005401618848, "cache_id": 459, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6679317951202393, "actual_norm": 3.6857073307037354, "relative_norm": 0.019386628533194793, "target_actual_norm": 3.6857073307037354, "match_error": 0.0, "scalar": 1.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.29303307759828, "first_log_odds": -16.5, "later_log_odds": 0.206966922401719, "first_differing_log_odds": -16.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-j.npz", "array_sha256": "90e1dabe099fcc5f44dc1437b62770b87ab956e818204ea89577fc3b318d8349", "seconds": 1.1977707501500845, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-opposite", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.663099765777588, "actual_norm": 3.685734987258911, "relative_norm": 0.019386774005234104, "target_actual_norm": 3.6857073307037354, "match_error": 2.765655517578125e-05, "scalar": 3.6630990795601974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 460, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-2.581181428775808, -0.14636129735904352], "logp": -2.7275427261348515, "cache_id": 461, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.663099765777588, "actual_norm": 3.685734987258911, "relative_norm": 0.019386774005234104, "target_actual_norm": 3.6857073307037354, "match_error": 2.765655517578125e-05, "scalar": 3.6630990795601974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.081181428775807, -1.0831052748810657e-05], "logp": -20.081192259828555, "cache_id": 462, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.663099765777588, "actual_norm": 3.685734987258911, "relative_norm": 0.019386774005234104, "target_actual_norm": 3.6857073307037354, "match_error": 2.765655517578125e-05, "scalar": 3.6630990795601974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.353649533693705, "first_log_odds": -17.5, "later_log_odds": 0.1463504663062949, "first_differing_log_odds": -17.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-opposite.npz", "array_sha256": "e1ea30c9b39cdd2127479d14711a5b3e835c3d268304fcf66f632d9e5d0116fb", "seconds": 1.1958749578334391, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-unrelated", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.671872615814209, "actual_norm": 3.6856675148010254, "relative_norm": 0.019386419103620958, "target_actual_norm": 3.6857073307037354, "match_error": -3.981590270996094e-05, "scalar": 3.671872430750227, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 463, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.288984809940096, -0.18456823472217795], "logp": -3.4735530446622738, "cache_id": 464, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.671872615814209, "actual_norm": 3.6856675148010254, "relative_norm": 0.019386419103620958, "target_actual_norm": 3.6857073307037354, "match_error": -3.981590270996094e-05, "scalar": 3.671872430750227, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.038984809940096, -7.537913212309403e-06], "logp": -20.03899234785331, "cache_id": 465, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.671872615814209, "actual_norm": 3.6856675148010254, "relative_norm": 0.019386419103620958, "target_actual_norm": 3.6857073307037354, "match_error": -3.981590270996094e-05, "scalar": 3.671872430750227, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.565439303191035, "first_log_odds": -16.75, "later_log_odds": 0.18456069680896547, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-unrelated.npz", "array_sha256": "459e8218a5537f0240a9fae7f9cb430b7005be3cbf445919159f69534dfb784e", "seconds": 1.1799674169160426, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-ordinary", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6811065673828125, "actual_norm": 3.6856889724731445, "relative_norm": 0.019386531969858355, "target_actual_norm": 3.6857073307037354, "match_error": -1.8358230590820312e-05, "scalar": 3.6811062422791565, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 466, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.5305897404575397, -0.18422985366693395], "logp": -3.7148195941244735, "cache_id": 467, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6811065673828125, "actual_norm": 3.6856889724731445, "relative_norm": 0.019386531969858355, "target_actual_norm": 3.6857073307037354, "match_error": -1.8358230590820312e-05, "scalar": 3.6811062422791565, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-19.90558974045754, -9.270836350782262e-06], "logp": -19.90559901129389, "cache_id": 468, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6811065673828125, "actual_norm": 3.6856889724731445, "relative_norm": 0.019386531969858355, "target_actual_norm": 3.6857073307037354, "match_error": -1.8358230590820312e-05, "scalar": 3.6811062422791565, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.190779417169416, "first_log_odds": -16.375, "later_log_odds": 0.18422058283058362, "first_differing_log_odds": -16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-ordinary.npz", "array_sha256": "8f0cfd6059a6ef5785c8015ea9584829388070d74b779a2896a0e4627680f2a1", "seconds": 1.194679332897067, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-random0", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6866252422332764, "actual_norm": 3.6856768131256104, "relative_norm": 0.01938646801232383, "target_actual_norm": 3.6857073307037354, "match_error": -3.0517578125e-05, "scalar": 3.6866256151582206, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 469, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.04971711893672, -0.16474201241189299], "logp": -3.2144591313486126, "cache_id": 470, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6866252422332764, "actual_norm": 3.6856768131256104, "relative_norm": 0.01938646801232383, "target_actual_norm": 3.6857073307037354, "match_error": -3.0517578125e-05, "scalar": 3.6866256151582206, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-20.17471711893672, -9.972674994296704e-06], "logp": -20.174727091611715, "cache_id": 471, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6866252422332764, "actual_norm": 3.6856768131256104, "relative_norm": 0.01938646801232383, "target_actual_norm": 3.6857073307037354, "match_error": -3.0517578125e-05, "scalar": 3.6866256151582206, "random_seed": 20271708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.960267960263103, "first_log_odds": -17.125, "later_log_odds": 0.1647320397368972, "first_differing_log_odds": -17.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-random0.npz", "array_sha256": "64471b87534be4e67dc22b47436941a01b3a28c9a4ca4370a88fc983d95545bf", "seconds": 1.1972725000232458, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-random1", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6752474308013916, "actual_norm": 3.6858301162719727, "relative_norm": 0.01938727437888656, "target_actual_norm": 3.6857073307037354, "match_error": 0.0001227855682373047, "scalar": 3.675246796592546, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 472, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-2.813485597768512, -0.18416332130712593], "logp": -2.997648919075638, "cache_id": 473, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6752474308013916, "actual_norm": 3.6858301162719727, "relative_norm": 0.01938727437888656, "target_actual_norm": 3.6857073307037354, "match_error": 0.0001227855682373047, "scalar": 3.675246796592546, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-19.563485597768512, -8.706200698313775e-06], "logp": -19.56349430396921, "cache_id": 474, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6752474308013916, "actual_norm": 3.6858301162719727, "relative_norm": 0.01938727437888656, "target_actual_norm": 3.6857073307037354, "match_error": 0.0001227855682373047, "scalar": 3.675246796592546, "random_seed": 20271709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.565845384893574, "first_log_odds": -16.75, "later_log_odds": 0.18415461510642572, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-random1.npz", "array_sha256": "6b8daf7079016d049b7bf4553eb9a8cd531fc630d4648cd398a67c81e61d0c02", "seconds": 1.1921418751589954, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-random2", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6865222454071045, "actual_norm": 3.685708999633789, "relative_norm": 0.019386637311679923, "target_actual_norm": 3.6857073307037354, "match_error": 1.6689300537109375e-06, "scalar": 3.6865228024489625, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 475, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-3.0498263335779567, -0.18473176763157204], "logp": -3.2345581012095286, "cache_id": 476, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6865222454071045, "actual_norm": 3.685708999633789, "relative_norm": 0.019386637311679923, "target_actual_norm": 3.6857073307037354, "match_error": 1.6689300537109375e-06, "scalar": 3.6865228024489625, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-19.674826333577958, -6.790168714943411e-06], "logp": -19.674833123746673, "cache_id": 477, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 190.11595153808594, "strength_one_norm": 3.6679317951202393, "target_shrink_scale": 1.0, "requested_norm": 3.6865222454071045, "actual_norm": 3.685708999633789, "relative_norm": 0.019386637311679923, "target_actual_norm": 3.6857073307037354, "match_error": 1.6689300537109375e-06, "scalar": 3.6865228024489625, "random_seed": 20271710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.440275022537143, "first_log_odds": -16.625, "later_log_odds": 0.18472497746285654, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-random2.npz", "array_sha256": "f7df4eb43e9089b3bb0f78eec15770edd35fffae2ebd73bbbbdd2bd9bc6b97fc", "seconds": 1.1971809170208871, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-08-paraphrase-bridge", "case_id": "eval-08", "city": "Brasilia", "country": "Brazil", "code": "BRL", "target_country": "Mexico", "target_code": "MXN", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.<|im_end|>", "output_text": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "output_ids": [785, 11413, 1483, 304, 61124, 24078, 374, 279, 34983, 8800, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 4728, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 414.19769287109375, 334.0015869140625, 3.0309908390045166, 63.8824348449707, 72.39556884765625, 166.64382934570312, 2.869385004043579, 23.720491409301758, 60.9614143371582, 205.03097534179688, 198.10629272460938, 51.50199508666992, 90.15103149414062, 95.5638656616211, 18.90890121459961, 319.4017028808594, 417.064453125, 159.3025360107422, 137.62925720214844, 510.7371520996094, 305.33819580078125, 378.42205810546875, 83.652587890625, 520.769775390625]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 414.1408996582031, 333.9892272949219, 3.1866490840911865, 63.92384719848633, 72.43738555908203, 166.5796661376953, 2.9467973709106445, 23.796632766723633, 60.981590270996094, 205.06558227539062, 198.065185546875, 51.4376106262207, 90.17338562011719, 95.59467315673828, 18.99370002746582, 319.40576171875, 417.0685729980469, 159.33154296875, 137.6737060546875, 510.7200622558594, 305.3363952636719, 378.4205627441406, 83.69444274902344, 520.744873046875]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.33415940403938293, 0.3447319567203522, 0.0031788016203790903, 0.058205001056194305, 0.0681990534067154, 0.1523953527212143, 0.0029082067776471376, 0.02050521969795227, 0.057868823409080505, 0.19420062005519867, 0.20150455832481384, 0.05274349823594093, 0.08997710794210434, 0.08971850574016571, 0.020285362377762794, 0.36183100938796997, 0.4877845346927643, 0.15584221482276917, 0.14678223431110382, 0.4472702443599701, 0.2886393368244171, 0.39767172932624817, 0.08031556010246277, 0.53376305103302]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 478, "cached": true, "scoring": {"normalized": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **BRL**.", "bare_format": false, "malformed": true, "first_code": "BRL", "all_codes": ["BRL"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Brasilia is the Brazilian Real, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Brazilian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [33, 4728], "tokens": ["B", "RL"], "log_probs": [-25.007148370099532, -0.15499518281724856], "logp": -25.16214355291678, "cache_id": 479, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 414.19769287109375, 334.0015869140625, 3.0309908390045166, 63.8824348449707, 72.39556884765625, 166.64382934570312, 2.869385004043579, 23.720491409301758, 60.9614143371582, 205.03097534179688, 198.10629272460938, 51.50199508666992, 90.15103149414062, 95.5638656616211, 18.90890121459961, 319.4017028808594, 417.064453125, 159.3025360107422, 137.62925720214844, 510.7371520996094, 305.33819580078125, 378.42205810546875, 83.652587890625, 520.769775390625]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 414.1408996582031, 333.9892272949219, 3.1866490840911865, 63.92384719848633, 72.43738555908203, 166.5796661376953, 2.9467973709106445, 23.796632766723633, 60.981590270996094, 205.06558227539062, 198.065185546875, 51.4376106262207, 90.17338562011719, 95.59467315673828, 18.99370002746582, 319.40576171875, 417.0685729980469, 159.33154296875, 137.6737060546875, 510.7200622558594, 305.3363952636719, 378.4205627441406, 83.69444274902344, 520.744873046875]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.33415940403938293, 0.3447319567203522, 0.0031788016203790903, 0.058205001056194305, 0.0681990534067154, 0.1523953527212143, 0.0029082067776471376, 0.02050521969795227, 0.057868823409080505, 0.19420062005519867, 0.20150455832481384, 0.05274349823594093, 0.08997710794210434, 0.08971850574016571, 0.020285362377762794, 0.36183100938796997, 0.4877845346927643, 0.15584221482276917, 0.14678223431110382, 0.4472702443599701, 0.2886393368244171, 0.39767172932624817, 0.08031556010246277, 0.53376305103302]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-9.382148370099532, -3.292795896373978e-05], "logp": -9.382181298058496, "cache_id": 480, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[457.69500732421875, 7.206311225891113, 0.41418027877807617, 42.67460250854492, 47.83418655395508, 4.917492866516113, 93.7812271118164, 21.46711540222168, 414.19769287109375, 334.0015869140625, 3.0309908390045166, 63.8824348449707, 72.39556884765625, 166.64382934570312, 2.869385004043579, 23.720491409301758, 60.9614143371582, 205.03097534179688, 198.10629272460938, 51.50199508666992, 90.15103149414062, 95.5638656616211, 18.90890121459961, 319.4017028808594, 417.064453125, 159.3025360107422, 137.62925720214844, 510.7371520996094, 305.33819580078125, 378.42205810546875, 83.652587890625, 520.769775390625]], "actual_norm": [[457.6947326660156, 7.271601676940918, 0.31960245966911316, 42.681819915771484, 47.83592224121094, 5.077962398529053, 93.77660369873047, 21.494007110595703, 414.1408996582031, 333.9892272949219, 3.1866490840911865, 63.92384719848633, 72.43738555908203, 166.5796661376953, 2.9467973709106445, 23.796632766723633, 60.981590270996094, 205.06558227539062, 198.065185546875, 51.4376106262207, 90.17338562011719, 95.59467315673828, 18.99370002746582, 319.40576171875, 417.0685729980469, 159.33154296875, 137.6737060546875, 510.7200622558594, 305.3363952636719, 378.4205627441406, 83.69444274902344, 520.744873046875]], "relative_norm": [[0.02671329490840435, 0.007713560946285725, 0.00035357996239326894, 0.03721408545970917, 0.04354385286569595, 0.004411408212035894, 0.08340384811162949, 0.019093260169029236, 0.33415940403938293, 0.3447319567203522, 0.0031788016203790903, 0.058205001056194305, 0.0681990534067154, 0.1523953527212143, 0.0029082067776471376, 0.02050521969795227, 0.057868823409080505, 0.19420062005519867, 0.20150455832481384, 0.05274349823594093, 0.08997710794210434, 0.08971850574016571, 0.020285362377762794, 0.36183100938796997, 0.4877845346927643, 0.15584221482276917, 0.14678223431110382, 0.4472702443599701, 0.2886393368244171, 0.39767172932624817, 0.08031556010246277, 0.53376305103302]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.779962254858285, "first_log_odds": 15.625, "later_log_odds": 0.15496225485828496, "first_differing_log_odds": 15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-08-paraphrase-bridge.npz", "array_sha256": "120d4991310da53dce3ac4614bbd0821c1cb0697cb8666fd5b56ef26478dbd8d", "seconds": 1.1949652503244579, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-baseline", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 481, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.2855027781929166, -3.8350730831250735e-07], "logp": -0.2855031617002249, "cache_id": 482, "trace": []}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.598002778192917, -0.21509406304693554], "logp": -19.813096841239854, "cache_id": 483, "trace": []}, "log_odds": -19.52759367953963, "first_log_odds": -19.3125, "later_log_odds": -0.2150936795396312, "first_differing_log_odds": -19.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-baseline.npz", "array_sha256": "ee3cab857c87f9feb5de07d053cf0dd9a31479221af0c909f1032c08456d2cc9", "seconds": 0.38032208336517215, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456780800, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-noop", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.993088722229004, "match_error": -9.993088722229004, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 484, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.2855027781929166, -3.8350730831250735e-07], "logp": -0.2855031617002249, "cache_id": 485, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.993088722229004, "match_error": -9.993088722229004, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.598002778192917, -0.21509406304693554], "logp": -19.813096841239854, "cache_id": 486, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.993088722229004, "match_error": -9.993088722229004, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.52759367953963, "first_log_odds": -19.3125, "later_log_odds": -0.2150936795396312, "first_differing_log_odds": -19.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-noop.npz", "array_sha256": "edfbeeeee0d39fe0ad34e53193c43add28f38ffd650395d80f127e5f297c92f9", "seconds": 0.3386262091808021, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-j", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.975058555603027, "actual_norm": 9.993088722229004, "relative_norm": 0.04999492059679963, "target_actual_norm": 9.993088722229004, "match_error": 0.0, "scalar": 0.16685646026347706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 487, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.7024202350090376, -4.602467713627431e-07], "logp": -0.702420695255809, "cache_id": 488, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.975058555603027, "actual_norm": 9.993088722229004, "relative_norm": 0.04999492059679963, "target_actual_norm": 9.993088722229004, "match_error": 0.0, "scalar": 0.16685646026347706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-18.327420235009036, -0.1633475337263771], "logp": -18.490767768735413, "cache_id": 489, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.975058555603027, "actual_norm": 9.993088722229004, "relative_norm": 0.04999492059679963, "target_actual_norm": 9.993088722229004, "match_error": 0.0, "scalar": 0.16685646026347706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.788347073479603, "first_log_odds": -17.625, "later_log_odds": -0.16334707347960276, "first_differing_log_odds": -17.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-j.npz", "array_sha256": "cd3f102dbbc900bcb1838e07e7189b7a5cad8c6b4ba7b1abb8a37c9884929a87", "seconds": 0.3148547918535769, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-opposite", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98066520690918, "actual_norm": 9.991790771484375, "relative_norm": 0.04998842701446287, "target_actual_norm": 9.993088722229004, "match_error": -0.0012979507446289062, "scalar": 9.980661403948034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 490, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.16293541739004144, -3.411161648658752e-07], "logp": -0.1629357585062063, "cache_id": 491, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98066520690918, "actual_norm": 9.991790771484375, "relative_norm": 0.04998842701446287, "target_actual_norm": 9.993088722229004, "match_error": -0.0012979507446289062, "scalar": 9.980661403948034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-20.53793541739004, -0.2193399160143419], "logp": -20.757275333404383, "cache_id": 492, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98066520690918, "actual_norm": 9.991790771484375, "relative_norm": 0.04998842701446287, "target_actual_norm": 9.993088722229004, "match_error": -0.0012979507446289062, "scalar": 9.980661403948034, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.59433957489818, "first_log_odds": -20.375, "later_log_odds": -0.21933957489817857, "first_differing_log_odds": -20.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-opposite.npz", "array_sha256": "327248b724c6b34005357449cbe5e3ec020af1ce3302bfe3646660e6abb07162", "seconds": 0.3380654170177877, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-unrelated", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990153312683105, "actual_norm": 9.992985725402832, "relative_norm": 0.0499944053088552, "target_actual_norm": 9.993088722229004, "match_error": -0.000102996826171875, "scalar": 9.990153435395769, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 493, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.3183020256717981, -5.198227167665001e-07], "logp": -0.31830254549451487, "cache_id": 494, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990153312683105, "actual_norm": 9.992985725402832, "relative_norm": 0.0499944053088552, "target_actual_norm": 9.993088722229004, "match_error": -0.000102996826171875, "scalar": 9.990153435395769, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.505802025671798, -0.17799809991585272], "logp": -19.683800125587652, "cache_id": 495, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990153312683105, "actual_norm": 9.992985725402832, "relative_norm": 0.0499944053088552, "target_actual_norm": 9.993088722229004, "match_error": -0.000102996826171875, "scalar": 9.990153435395769, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.365497580093137, "first_log_odds": -19.1875, "later_log_odds": -0.1779975800931375, "first_differing_log_odds": -19.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-unrelated.npz", "array_sha256": "35758768a4668a10b901cbd63bd5f0f413e1be408517253b619cdcc1ccf6f648", "seconds": 0.3393488749861717, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456813568, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-ordinary", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98376750946045, "actual_norm": 9.993012428283691, "relative_norm": 0.04999453890202597, "target_actual_norm": 9.993088722229004, "match_error": -7.62939453125e-05, "scalar": 9.983768233518276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 496, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.31905119369621576, -4.1124810294153036e-07], "logp": -0.3190516049443187, "cache_id": 497, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98376750946045, "actual_norm": 9.993012428283691, "relative_norm": 0.04999453890202597, "target_actual_norm": 9.993088722229004, "match_error": -7.62939453125e-05, "scalar": 9.983768233518276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.319051193696215, -0.20378089572539543], "logp": -19.52283208942161, "cache_id": 498, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.98376750946045, "actual_norm": 9.993012428283691, "relative_norm": 0.04999453890202597, "target_actual_norm": 9.993088722229004, "match_error": -7.62939453125e-05, "scalar": 9.983768233518276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.20378048447729, "first_log_odds": -19.0, "later_log_odds": -0.20378048447728858, "first_differing_log_odds": -19.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-ordinary.npz", "array_sha256": "bb1a8b25a4aafe295742d6dbdfd85217573afdbad26a154d7857a2ca4ae40eb4", "seconds": 0.3407333749346435, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-random0", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.984474182128906, "actual_norm": 9.993244171142578, "relative_norm": 0.04999569829990094, "target_actual_norm": 9.993088722229004, "match_error": 0.00015544891357421875, "scalar": 9.984473464770417, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 499, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.22928001743912999, -4.1057343622981584e-07], "logp": -0.2292804280125662, "cache_id": 500, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.984474182128906, "actual_norm": 9.993244171142578, "relative_norm": 0.04999569829990094, "target_actual_norm": 9.993088722229004, "match_error": 0.00015544891357421875, "scalar": 9.984473464770417, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-20.04178001743913, -0.2003442907674637], "logp": -20.242124308206595, "cache_id": 501, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.984474182128906, "actual_norm": 9.993244171142578, "relative_norm": 0.04999569829990094, "target_actual_norm": 9.993088722229004, "match_error": 0.00015544891357421875, "scalar": 9.984473464770417, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.01284388019403, "first_log_odds": -19.8125, "later_log_odds": -0.20034388019402982, "first_differing_log_odds": -19.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-random0.npz", "array_sha256": "1a99ac24b9693920bdd9c1bf9e2ec37c6fcb006b9de2bf58a588f931ab2c441c", "seconds": 0.3310665423050523, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-random1", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990494728088379, "actual_norm": 9.992959976196289, "relative_norm": 0.04999427648686909, "target_actual_norm": 9.993088722229004, "match_error": -0.00012874603271484375, "scalar": 9.990496520869783, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 502, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.2052519635627605, -3.6432627028801736e-07], "logp": -0.20525232788903078, "cache_id": 503, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990494728088379, "actual_norm": 9.992959976196289, "relative_norm": 0.04999427648686909, "target_actual_norm": 9.993088722229004, "match_error": -0.00012874603271484375, "scalar": 9.990496520869783, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-20.08025196356276, -0.18716023295344936], "logp": -20.267412196516208, "cache_id": 504, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.990494728088379, "actual_norm": 9.992959976196289, "relative_norm": 0.04999427648686909, "target_actual_norm": 9.993088722229004, "match_error": -0.00012874603271484375, "scalar": 9.990496520869783, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.062159868627177, "first_log_odds": -19.875, "later_log_odds": -0.1871598686271767, "first_differing_log_odds": -19.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-random1.npz", "array_sha256": "f5f4d4d3d3333c349e7f328daf00f7aa12ecdab00de685987e527e89c3d8c27a", "seconds": 0.34213108383119106, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-random2", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXN<|im_end|>", "output_text": "MXN", "output_ids": [29423, 45, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.9905366897583, "actual_norm": 9.993083953857422, "relative_norm": 0.049994896740876274, "target_actual_norm": 9.993088722229004, "match_error": -4.76837158203125e-06, "scalar": 9.990534641478007, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 505, "cached": true, "scoring": {"normalized": "MXN", "bare_format": true, "malformed": false, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-0.25644966605584457, -5.116243552305442e-07], "logp": -0.2564501776801998, "cache_id": 506, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.9905366897583, "actual_norm": 9.993083953857422, "relative_norm": 0.049994896740876274, "target_actual_norm": 9.993088722229004, "match_error": -4.76837158203125e-06, "scalar": 9.990534641478007, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.818949666055843, -0.20555650020676744], "logp": -20.02450616626261, "cache_id": 507, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 199.882080078125, "strength_one_norm": 59.78225326538086, "target_shrink_scale": 0.16685646026347706, "requested_norm": 9.9905366897583, "actual_norm": 9.993083953857422, "relative_norm": 0.049994896740876274, "target_actual_norm": 9.993088722229004, "match_error": -4.76837158203125e-06, "scalar": 9.990534641478007, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.76805598858241, "first_log_odds": -19.5625, "later_log_odds": -0.20555598858241098, "first_differing_log_odds": -19.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-random2.npz", "array_sha256": "70785d655cdd96dbce8db0fcd37415b5f37b7df7f6be4c3a958b6fa1c60a76e8", "seconds": 0.34092145785689354, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456846336, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-main-bridge", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 617.8638305664062, 895.3025512695312, 136.22972106933594, 7.872739791870117, 156.71578979492188, 265.1001892089844, 105.29069519042969, 9.382464408874512, 53.70658493041992, 66.7265853881836, 106.11268615722656, 16.959056854248047, 11.8460111618042, 47.36756134033203, 88.224853515625, 144.49066162109375, 50.74361038208008, 72.47496795654297, 260.5363464355469, 255.1626434326172, 255.36285400390625, 16.704078674316406, 603.4246215820312, 295.8453674316406, 610.4976196289062, 136.4521484375, 713.6134033203125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 617.9889526367188, 895.4339599609375, 136.22479248046875, 7.992937088012695, 156.7188720703125, 265.194091796875, 105.41090393066406, 9.445013999938965, 53.767311096191406, 66.80367279052734, 106.11836242675781, 17.01961898803711, 11.92503547668457, 47.4244499206543, 88.20458221435547, 144.42160034179688, 50.812843322753906, 72.52595520019531, 260.5640563964844, 255.2095184326172, 255.41981506347656, 16.804561614990234, 603.4994506835938, 295.9044189453125, 610.4370727539062, 136.46400451660156, 713.5335693359375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.501487135887146, 0.8810359239578247, 0.14166130125522614, 0.007847019471228123, 0.14562368392944336, 0.25537487864494324, 0.10062914341688156, 0.00828498974442482, 0.053127117455005646, 0.06371670216321945, 0.10838871449232101, 0.017542028799653053, 0.011994294822216034, 0.04544093459844589, 0.08315733075141907, 0.11637066304683685, 0.05056118592619896, 0.0805681049823761, 0.289837509393692, 0.3060944974422455, 0.25526246428489685, 0.017917167395353317, 0.5320923328399658, 0.2869502305984497, 0.6595017910003662, 0.132045179605484, 0.7030187249183655]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 508, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-35.214065481476624, -6.530016391631008e-07], "logp": -35.214066134478266, "cache_id": 509, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 617.8638305664062, 895.3025512695312, 136.22972106933594, 7.872739791870117, 156.71578979492188, 265.1001892089844, 105.29069519042969, 9.382464408874512, 53.70658493041992, 66.7265853881836, 106.11268615722656, 16.959056854248047, 11.8460111618042, 47.36756134033203, 88.224853515625, 144.49066162109375, 50.74361038208008, 72.47496795654297, 260.5363464355469, 255.1626434326172, 255.36285400390625, 16.704078674316406, 603.4246215820312, 295.8453674316406, 610.4976196289062, 136.4521484375, 713.6134033203125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 617.9889526367188, 895.4339599609375, 136.22479248046875, 7.992937088012695, 156.7188720703125, 265.194091796875, 105.41090393066406, 9.445013999938965, 53.767311096191406, 66.80367279052734, 106.11836242675781, 17.01961898803711, 11.92503547668457, 47.4244499206543, 88.20458221435547, 144.42160034179688, 50.812843322753906, 72.52595520019531, 260.5640563964844, 255.2095184326172, 255.41981506347656, 16.804561614990234, 603.4994506835938, 295.9044189453125, 610.4370727539062, 136.46400451660156, 713.5335693359375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.501487135887146, 0.8810359239578247, 0.14166130125522614, 0.007847019471228123, 0.14562368392944336, 0.25537487864494324, 0.10062914341688156, 0.00828498974442482, 0.053127117455005646, 0.06371670216321945, 0.10838871449232101, 0.017542028799653053, 0.011994294822216034, 0.04544093459844589, 0.08315733075141907, 0.11637066304683685, 0.05056118592619896, 0.0805681049823761, 0.289837509393692, 0.3060944974422455, 0.25526246428489685, 0.017917167395353317, 0.5320923328399658, 0.2869502305984497, 0.6595017910003662, 0.132045179605484, 0.7030187249183655]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.02461235647662036, -0.13102465993230394], "logp": -0.1556370164089243, "cache_id": 510, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 617.8638305664062, 895.3025512695312, 136.22972106933594, 7.872739791870117, 156.71578979492188, 265.1001892089844, 105.29069519042969, 9.382464408874512, 53.70658493041992, 66.7265853881836, 106.11268615722656, 16.959056854248047, 11.8460111618042, 47.36756134033203, 88.224853515625, 144.49066162109375, 50.74361038208008, 72.47496795654297, 260.5363464355469, 255.1626434326172, 255.36285400390625, 16.704078674316406, 603.4246215820312, 295.8453674316406, 610.4976196289062, 136.4521484375, 713.6134033203125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 617.9889526367188, 895.4339599609375, 136.22479248046875, 7.992937088012695, 156.7188720703125, 265.194091796875, 105.41090393066406, 9.445013999938965, 53.767311096191406, 66.80367279052734, 106.11836242675781, 17.01961898803711, 11.92503547668457, 47.4244499206543, 88.20458221435547, 144.42160034179688, 50.812843322753906, 72.52595520019531, 260.5640563964844, 255.2095184326172, 255.41981506347656, 16.804561614990234, 603.4994506835938, 295.9044189453125, 610.4370727539062, 136.46400451660156, 713.5335693359375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.501487135887146, 0.8810359239578247, 0.14166130125522614, 0.007847019471228123, 0.14562368392944336, 0.25537487864494324, 0.10062914341688156, 0.00828498974442482, 0.053127117455005646, 0.06371670216321945, 0.10838871449232101, 0.017542028799653053, 0.011994294822216034, 0.04544093459844589, 0.08315733075141907, 0.11637066304683685, 0.05056118592619896, 0.0805681049823761, 0.289837509393692, 0.3060944974422455, 0.25526246428489685, 0.017917167395353317, 0.5320923328399658, 0.2869502305984497, 0.6595017910003662, 0.132045179605484, 0.7030187249183655]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 35.05842911806934, "first_log_odds": 35.189453125, "later_log_odds": -0.13102400693065874, "first_differing_log_odds": 35.189453125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-main-bridge.npz", "array_sha256": "c07f4f448c8468954396ff7f4868c4393e0bb4bd6a6390d307ca1b1635043494", "seconds": 0.34170841705054045, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-baseline", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 511, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.500000042283407, -3.0219885974977382e-06], "logp": -22.500003064272004, "cache_id": 512, "trace": []}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.62500004228341, -0.2678957791858106], "logp": -33.892895821469224, "cache_id": 513, "trace": []}, "log_odds": -11.39289275719722, "first_log_odds": -11.125000000000004, "later_log_odds": -0.2678927571972167, "first_differing_log_odds": -11.125000000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-baseline.npz", "array_sha256": "e90342c05e5c9460ac3f68bfc9e332d852edad530f126df330f7984b0e08f992", "seconds": 1.145898665767163, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-noop", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.919777870178223, "match_error": -9.919777870178223, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 514, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.500000042283407, -3.0219885974977382e-06], "logp": -22.500003064272004, "cache_id": 515, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.919777870178223, "match_error": -9.919777870178223, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.62500004228341, -0.2678957791858106], "logp": -33.892895821469224, "cache_id": 516, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.919777870178223, "match_error": -9.919777870178223, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.39289275719722, "first_log_odds": -11.125000000000004, "later_log_odds": -0.2678927571972167, "first_differing_log_odds": -11.125000000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-noop.npz", "array_sha256": "08fd456aee39c316b8d793a1e95fd72a35ac3fc4e5cfbfb7a5c311fab0f3c7aa", "seconds": 1.1662052921019495, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-j", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 17692, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.907428741455078, "actual_norm": 9.919777870178223, "relative_norm": 0.049999870213123514, "target_actual_norm": 9.919777870178223, "match_error": 0.0, "scalar": 0.16564712673018928, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 517, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.12500002605024, -1.6626347641377315e-06], "logp": -23.125001688685007, "cache_id": 518, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.907428741455078, "actual_norm": 9.919777870178223, "relative_norm": 0.049999870213123514, "target_actual_norm": 9.919777870178223, "match_error": 0.0, "scalar": 0.16564712673018928, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-32.81250002605024, -0.22261319956132758], "logp": -33.03511322561157, "cache_id": 519, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.907428741455078, "actual_norm": 9.919777870178223, "relative_norm": 0.049999870213123514, "target_actual_norm": 9.919777870178223, "match_error": 0.0, "scalar": 0.16564712673018928, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.91011153692656, "first_log_odds": -9.687499999999996, "later_log_odds": -0.2226115369265642, "first_differing_log_odds": -9.687499999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-j.npz", "array_sha256": "0973b05996894a503eb3f0dd0bab487996d2768ed0b912510b4827413fa08135", "seconds": 1.1636037081480026, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-opposite", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.90373420715332, "actual_norm": 9.919772148132324, "relative_norm": 0.049999841371595403, "target_actual_norm": 9.919777870178223, "match_error": -5.7220458984375e-06, "scalar": 9.903733307583025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 520, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.62500004778079, -2.795603133936976e-06], "logp": -22.625002843383925, "cache_id": 521, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.90373420715332, "actual_norm": 9.919772148132324, "relative_norm": 0.049999841371595403, "target_actual_norm": 9.919777870178223, "match_error": -5.7220458984375e-06, "scalar": 9.903733307583025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.93750004778079, -0.2971450934864453], "logp": -34.23464514126723, "cache_id": 522, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.90373420715332, "actual_norm": 9.919772148132324, "relative_norm": 0.049999841371595403, "target_actual_norm": 9.919777870178223, "match_error": -5.7220458984375e-06, "scalar": 9.903733307583025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.609642297883305, "first_log_odds": -11.312499999999996, "later_log_odds": -0.29714229788330826, "first_differing_log_odds": -11.312499999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-opposite.npz", "array_sha256": "e68744d7f773bc9524ed2620b798b62bdbf2ad153ceac19016cf2c61ff46ec27", "seconds": 1.1611445001326501, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-unrelated", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 17692, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.905699729919434, "actual_norm": 9.919489860534668, "relative_norm": 0.04999841852287537, "target_actual_norm": 9.919777870178223, "match_error": -0.0002880096435546875, "scalar": 9.905701036957907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 523, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.12500003256203, -3.493663772992156e-06], "logp": -23.125003526225804, "cache_id": 524, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.905699729919434, "actual_norm": 9.919489860534668, "relative_norm": 0.04999841852287537, "target_actual_norm": 9.919777870178223, "match_error": -0.0002880096435546875, "scalar": 9.905701036957907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.68750003256203, -0.25485442826423155], "logp": -33.94235446082626, "cache_id": 525, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.905699729919434, "actual_norm": 9.919489860534668, "relative_norm": 0.04999841852287537, "target_actual_norm": 9.919777870178223, "match_error": -0.0002880096435546875, "scalar": 9.905701036957907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.817350934600455, "first_log_odds": -10.562499999999996, "later_log_odds": -0.2548509346004586, "first_differing_log_odds": -10.562499999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-unrelated.npz", "array_sha256": "ba0b572c8828afac7b5f2972f62659b1a86547020cf801c720b187ff76ecdc7a", "seconds": 1.1570958751253784, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-ordinary", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.914177894592285, "actual_norm": 9.919758796691895, "relative_norm": 0.04999977407469648, "target_actual_norm": 9.919777870178223, "match_error": -1.9073486328125e-05, "scalar": 9.91417740964971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 526, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.00000003343649, -3.833793635013773e-06], "logp": -23.000003867230124, "cache_id": 527, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.914177894592285, "actual_norm": 9.919758796691895, "relative_norm": 0.04999977407469648, "target_actual_norm": 9.919777870178223, "match_error": -1.9073486328125e-05, "scalar": 9.91417740964971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.37500003343649, -0.24467162224745934], "logp": -33.61967165568395, "cache_id": 528, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.914177894592285, "actual_norm": 9.919758796691895, "relative_norm": 0.04999977407469648, "target_actual_norm": 9.919777870178223, "match_error": -1.9073486328125e-05, "scalar": 9.91417740964971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.619667788453828, "first_log_odds": -10.375000000000004, "later_log_odds": -0.24466778845382464, "first_differing_log_odds": -10.375000000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-ordinary.npz", "array_sha256": "34ec6979a7b7c4869c23ec350f4934a3acd69d3ddb2d96959c6fff62d28a9c7e", "seconds": 1.165255915839225, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-random0", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 17692, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915446281433105, "actual_norm": 9.919560432434082, "relative_norm": 0.04999877423505538, "target_actual_norm": 9.919777870178223, "match_error": -0.000217437744140625, "scalar": 9.915445081458529, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 529, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.250000032579074, -2.74866604749517e-06], "logp": -23.250002781245122, "cache_id": 530, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915446281433105, "actual_norm": 9.919560432434082, "relative_norm": 0.04999877423505538, "target_actual_norm": 9.919777870178223, "match_error": -0.000217437744140625, "scalar": 9.915445081458529, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.750000032579074, -0.257274856801239], "logp": -34.00727488938031, "cache_id": 531, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915446281433105, "actual_norm": 9.919560432434082, "relative_norm": 0.04999877423505538, "target_actual_norm": 9.919777870178223, "match_error": -0.000217437744140625, "scalar": 9.915445081458529, "random_seed": 20271808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.75727210813519, "first_log_odds": -10.5, "later_log_odds": -0.2572721081351901, "first_differing_log_odds": -10.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-random0.npz", "array_sha256": "dc9fbe41a126be36ffb53c7731441dbe7d2e415e8712a8a86a99b40dc5114f57", "seconds": 1.1797843747772276, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-random1", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.922802925109863, "actual_norm": 9.919431686401367, "relative_norm": 0.04999812530067293, "target_actual_norm": 9.919777870178223, "match_error": -0.00034618377685546875, "scalar": 9.92280514613958, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 532, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.875000033755946, -2.356622136599339e-06], "logp": -22.875002390378082, "cache_id": 533, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.922802925109863, "actual_norm": 9.919431686401367, "relative_norm": 0.04999812530067293, "target_actual_norm": 9.919777870178223, "match_error": -0.00034618377685546875, "scalar": 9.92280514613958, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.812500033755946, -0.2874529724333763], "logp": -34.099953006189324, "cache_id": 534, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.922802925109863, "actual_norm": 9.919431686401367, "relative_norm": 0.04999812530067293, "target_actual_norm": 9.919777870178223, "match_error": -0.00034618377685546875, "scalar": 9.92280514613958, "random_seed": 20271809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.224950615811242, "first_log_odds": -10.9375, "later_log_odds": -0.28745061581124176, "first_differing_log_odds": -10.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-random1.npz", "array_sha256": "63af3446e8a6b3f72b6f16a315159fef3a8224aee81c749ef69f66cbcd699280", "seconds": 1.1619839160703123, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-random2", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915994644165039, "actual_norm": 9.919697761535645, "relative_norm": 0.04999946643172999, "target_actual_norm": 9.919777870178223, "match_error": -8.0108642578125e-05, "scalar": 9.915993775226525, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 535, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.25000003841757, -3.2338961474170597e-06], "logp": -22.25000327231372, "cache_id": 536, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915994644165039, "actual_norm": 9.919697761535645, "relative_norm": 0.04999946643172999, "target_actual_norm": 9.919777870178223, "match_error": -8.0108642578125e-05, "scalar": 9.915993775226525, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-33.25000003841757, -0.2978701361609485], "logp": -33.54787017457851, "cache_id": 537, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 198.3960723876953, "strength_one_norm": 59.810455322265625, "target_shrink_scale": 0.16564712673018928, "requested_norm": 9.915994644165039, "actual_norm": 9.919697761535645, "relative_norm": 0.04999946643172999, "target_actual_norm": 9.919777870178223, "match_error": -8.0108642578125e-05, "scalar": 9.915993775226525, "random_seed": 20271810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.297866902264794, "first_log_odds": -10.999999999999996, "later_log_odds": -0.29786690226479706, "first_differing_log_odds": -10.999999999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-random2.npz", "array_sha256": "c97eee1d4748c670816000828dd3f0fccdbacf94ce4ee8c5daa89e942a5d2ec7", "seconds": 1.1598662920296192, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-09-paraphrase-bridge", "case_id": "eval-09", "city": "Cancun", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.<|im_end|>", "output_text": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "output_ids": [785, 11413, 1483, 304, 80323, 359, 374, 279, 2274, 42364, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26749, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 624.2059936523438, 880.0093994140625, 223.4894256591797, 126.49742889404297, 270.2008361816406, 207.08387756347656, 88.56124877929688, 7.467529773712158, 48.906471252441406, 67.2015380859375, 104.95238494873047, 36.09766387939453, 10.337225914001465, 8.916204452514648, 144.1511688232422, 232.46690368652344, 280.2255859375, 283.3988342285156, 15.8804349899292, 500.56787109375, 283.5205383300781, 551.3009643554688, 147.10243225097656, 681.2431640625]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 624.1004028320312, 879.9856567382812, 223.5234832763672, 126.53014373779297, 270.1712646484375, 207.13973999023438, 88.5800552368164, 7.540553569793701, 48.89158248901367, 67.23731994628906, 104.96161651611328, 36.171226501464844, 10.402076721191406, 9.022254943847656, 144.16050720214844, 232.49412536621094, 280.2104797363281, 283.3752136230469, 15.990073204040527, 500.4749450683594, 283.533447265625, 551.3327026367188, 147.14344787597656, 681.2661743164062]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.5006458163261414, 0.8691034317016602, 0.22080248594284058, 0.11573518067598343, 0.253301739692688, 0.19044339656829834, 0.08530790358781815, 0.006485274061560631, 0.04667183384299278, 0.0629769116640091, 0.10584773123264313, 0.03676733002066612, 0.010150076821446419, 0.008552366867661476, 0.15181711316108704, 0.2620777189731598, 0.32232344150543213, 0.27808529138565063, 0.017096392810344696, 0.4396381974220276, 0.2691400349140167, 0.5797839760780334, 0.14163585007190704, 0.7256135940551758]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 538, "cached": true, "scoring": {"normalized": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **USD**.", "bare_format": false, "malformed": true, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Cancun is the US Dollar, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-43.64037132436034, -1.4176824160574011e-05], "logp": -43.6403855011845, "cache_id": 539, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 624.2059936523438, 880.0093994140625, 223.4894256591797, 126.49742889404297, 270.2008361816406, 207.08387756347656, 88.56124877929688, 7.467529773712158, 48.906471252441406, 67.2015380859375, 104.95238494873047, 36.09766387939453, 10.337225914001465, 8.916204452514648, 144.1511688232422, 232.46690368652344, 280.2255859375, 283.3988342285156, 15.8804349899292, 500.56787109375, 283.5205383300781, 551.3009643554688, 147.10243225097656, 681.2431640625]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 624.1004028320312, 879.9856567382812, 223.5234832763672, 126.53014373779297, 270.1712646484375, 207.13973999023438, 88.5800552368164, 7.540553569793701, 48.89158248901367, 67.23731994628906, 104.96161651611328, 36.171226501464844, 10.402076721191406, 9.022254943847656, 144.16050720214844, 232.49412536621094, 280.2104797363281, 283.3752136230469, 15.990073204040527, 500.4749450683594, 283.533447265625, 551.3327026367188, 147.14344787597656, 681.2661743164062]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.5006458163261414, 0.8691034317016602, 0.22080248594284058, 0.11573518067598343, 0.253301739692688, 0.19044339656829834, 0.08530790358781815, 0.006485274061560631, 0.04667183384299278, 0.0629769116640091, 0.10584773123264313, 0.03676733002066612, 0.010150076821446419, 0.008552366867661476, 0.15181711316108704, 0.2620777189731598, 0.32232344150543213, 0.27808529138565063, 0.017096392810344696, 0.4396381974220276, 0.2691400349140167, 0.5797839760780334, 0.14163585007190704, 0.7256135940551758]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-5.015371324360338, -0.340844043841244], "logp": -5.356215368201582, "cache_id": 540, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 624.2059936523438, 880.0093994140625, 223.4894256591797, 126.49742889404297, 270.2008361816406, 207.08387756347656, 88.56124877929688, 7.467529773712158, 48.906471252441406, 67.2015380859375, 104.95238494873047, 36.09766387939453, 10.337225914001465, 8.916204452514648, 144.1511688232422, 232.46690368652344, 280.2255859375, 283.3988342285156, 15.8804349899292, 500.56787109375, 283.5205383300781, 551.3009643554688, 147.10243225097656, 681.2431640625]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 624.1004028320312, 879.9856567382812, 223.5234832763672, 126.53014373779297, 270.1712646484375, 207.13973999023438, 88.5800552368164, 7.540553569793701, 48.89158248901367, 67.23731994628906, 104.96161651611328, 36.171226501464844, 10.402076721191406, 9.022254943847656, 144.16050720214844, 232.49412536621094, 280.2104797363281, 283.3752136230469, 15.990073204040527, 500.4749450683594, 283.533447265625, 551.3327026367188, 147.14344787597656, 681.2661743164062]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.5006458163261414, 0.8691034317016602, 0.22080248594284058, 0.11573518067598343, 0.253301739692688, 0.19044339656829834, 0.08530790358781815, 0.006485274061560631, 0.04667183384299278, 0.0629769116640091, 0.10584773123264313, 0.03676733002066612, 0.010150076821446419, 0.008552366867661476, 0.15181711316108704, 0.2620777189731598, 0.32232344150543213, 0.27808529138565063, 0.017096392810344696, 0.4396381974220276, 0.2691400349140167, 0.5797839760780334, 0.14163585007190704, 0.7256135940551758]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 38.284170132982915, "first_log_odds": 38.625, "later_log_odds": -0.34082986701708506, "first_differing_log_odds": 38.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-09-paraphrase-bridge.npz", "array_sha256": "37e108ca46c78ed629e5e1d29d8843aba24d3029ac03637f96ef3052448ebacb", "seconds": 1.1576168751344085, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-baseline", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 541, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.1517051200841848, -6.521311944310959e-08], "logp": -1.1517051852973041, "cache_id": 542, "trace": []}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.089205120084184, -1.9144194853940626], "logp": -21.003624605478247, "cache_id": 543, "trace": []}, "log_odds": -19.851919420180945, "first_log_odds": -17.9375, "later_log_odds": -1.9144194201809448, "first_differing_log_odds": -17.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-baseline.npz", "array_sha256": "f584ce4957dd58b49b0f8e9c21b140ab4a98ac1cdafdcf349651810d20dc1e29", "seconds": 0.38344379095360637, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-noop", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.231122970581055, "match_error": -9.231122970581055, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 544, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.1517051200841848, -6.521311944310959e-08], "logp": -1.1517051852973041, "cache_id": 545, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.231122970581055, "match_error": -9.231122970581055, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.089205120084184, -1.9144194853940626], "logp": -21.003624605478247, "cache_id": 546, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.231122970581055, "match_error": -9.231122970581055, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.851919420180945, "first_log_odds": -17.9375, "later_log_odds": -1.9144194201809448, "first_differing_log_odds": -17.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-noop.npz", "array_sha256": "daf2a3b86639a9cd24491cf231e5f5f7c7df3e783cdc73e0985a633e30d60bce", "seconds": 0.3395526660606265, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-j", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22697639465332, "actual_norm": 9.231122970581055, "relative_norm": 0.04999977065116803, "target_actual_norm": 9.231122970581055, "match_error": 0.0, "scalar": 0.19169541446361588, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 547, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.012872091108131, -7.163529446177576e-08], "logp": -1.0128721627434254, "cache_id": 548, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22697639465332, "actual_norm": 9.231122970581055, "relative_norm": 0.04999977065116803, "target_actual_norm": 9.231122970581055, "match_error": 0.0, "scalar": 0.19169541446361588, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.575372091108132, -1.5533984468408226], "logp": -18.128770537948956, "cache_id": 549, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22697639465332, "actual_norm": 9.231122970581055, "relative_norm": 0.04999977065116803, "target_actual_norm": 9.231122970581055, "match_error": 0.0, "scalar": 0.19169541446361588, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.115898375205532, "first_log_odds": -15.562500000000002, "later_log_odds": -1.5533983752055303, "first_differing_log_odds": -15.562500000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-j.npz", "array_sha256": "2a5bcd34410aff6ab7d8454ab7b4e595dfd37db8f700674174d32f067ca25644", "seconds": 0.3385799159295857, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-opposite", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.229573249816895, "actual_norm": 9.231159210205078, "relative_norm": 0.049999966940708906, "target_actual_norm": 9.231122970581055, "match_error": 3.62396240234375e-05, "scalar": 9.229573557240656, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 550, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.417020185770716, -6.024397532554914e-08], "logp": -1.4170202460146915, "cache_id": 551, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.229573249816895, "actual_norm": 9.231159210205078, "relative_norm": 0.049999966940708906, "target_actual_norm": 9.231122970581055, "match_error": 3.62396240234375e-05, "scalar": 9.229573557240656, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.729520185770717, -2.1241758195805343], "logp": -21.853696005351253, "cache_id": 552, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.229573249816895, "actual_norm": 9.231159210205078, "relative_norm": 0.049999966940708906, "target_actual_norm": 9.231122970581055, "match_error": 3.62396240234375e-05, "scalar": 9.229573557240656, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.436675759336563, "first_log_odds": -18.3125, "later_log_odds": -2.1241757593365627, "first_differing_log_odds": -18.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-opposite.npz", "array_sha256": "0f2eec5d39ac67ade9bd393a9d09892467cdfd42f251256e4b6d5c9864a6ed87", "seconds": 0.3384090829640627, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-unrelated", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.224361419677734, "actual_norm": 9.230497360229492, "relative_norm": 0.04999638207383082, "target_actual_norm": 9.231122970581055, "match_error": -0.0006256103515625, "scalar": 9.224361894186586, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 553, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.1607136820200945, -8.56165082150111e-08], "logp": -1.1607137676366026, "cache_id": 554, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.224361419677734, "actual_norm": 9.230497360229492, "relative_norm": 0.04999638207383082, "target_actual_norm": 9.231122970581055, "match_error": -0.0006256103515625, "scalar": 9.224361894186586, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-18.035713682020095, -1.7749744781132062], "logp": -19.810688160133303, "cache_id": 555, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.224361419677734, "actual_norm": 9.230497360229492, "relative_norm": 0.04999638207383082, "target_actual_norm": 9.231122970581055, "match_error": -0.0006256103515625, "scalar": 9.224361894186586, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.649974392496702, "first_log_odds": -16.875, "later_log_odds": -1.774974392496702, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-unrelated.npz", "array_sha256": "ba5513b27a842db7099cd2396ab35e9d58117497900c3cb0bdf4c186386f18a3", "seconds": 0.3404512503184378, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-ordinary", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.232390403747559, "actual_norm": 9.231164932250977, "relative_norm": 0.0499999979337943, "target_actual_norm": 9.231122970581055, "match_error": 4.1961669921875e-05, "scalar": 9.232390672405018, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 556, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.2365101989398584, -8.211998943398273e-08], "logp": -1.2365102810598478, "cache_id": 557, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.232390403747559, "actual_norm": 9.231164932250977, "relative_norm": 0.0499999979337943, "target_actual_norm": 9.231122970581055, "match_error": 4.1961669921875e-05, "scalar": 9.232390672405018, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.174010198939857, -2.002447075642637], "logp": -21.176457274582496, "cache_id": 558, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.232390403747559, "actual_norm": 9.231164932250977, "relative_norm": 0.0499999979337943, "target_actual_norm": 9.231122970581055, "match_error": 4.1961669921875e-05, "scalar": 9.232390672405018, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.93994699352265, "first_log_odds": -17.9375, "later_log_odds": -2.00244699352265, "first_differing_log_odds": -17.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-ordinary.npz", "array_sha256": "13f9ecd705af3f5b7e00c28adaa0bafd699ddc72f50a9230dd36173ce8d67c96", "seconds": 0.3429097500629723, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-random0", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22302532196045, "actual_norm": 9.230902671813965, "relative_norm": 0.04999857741738008, "target_actual_norm": 9.231122970581055, "match_error": -0.00022029876708984375, "scalar": 9.223023764483514, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 559, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.3255891915758031, -7.024335022294952e-08], "logp": -1.3255892618191534, "cache_id": 560, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22302532196045, "actual_norm": 9.230902671813965, "relative_norm": 0.04999857741738008, "target_actual_norm": 9.231122970581055, "match_error": -0.00022029876708984375, "scalar": 9.223023764483514, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.138089191575805, -1.8106065150260788], "logp": -20.948695706601885, "cache_id": 561, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.22302532196045, "actual_norm": 9.230902671813965, "relative_norm": 0.04999857741738008, "target_actual_norm": 9.231122970581055, "match_error": -0.00022029876708984375, "scalar": 9.223023764483514, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.62310644478273, "first_log_odds": -17.8125, "later_log_odds": -1.8106064447827315, "first_differing_log_odds": -17.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-random0.npz", "array_sha256": "c2879d13bf823a7a1d5460324c6e71572ad887ec69963dafda486a4a3f39c01d", "seconds": 0.3383840420283377, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-random1", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.230277061462402, "actual_norm": 9.231096267700195, "relative_norm": 0.049999626016769494, "target_actual_norm": 9.231122970581055, "match_error": -2.6702880859375e-05, "scalar": 9.230277836031746, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 562, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.4157655541027403, -6.951827005552956e-08], "logp": -1.4157656236210103, "cache_id": 563, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.230277061462402, "actual_norm": 9.231096267700195, "relative_norm": 0.049999626016769494, "target_actual_norm": 9.231122970581055, "match_error": -2.6702880859375e-05, "scalar": 9.230277836031746, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.16576555410274, -2.170141928769987], "logp": -21.335907482872727, "cache_id": 564, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.230277061462402, "actual_norm": 9.231096267700195, "relative_norm": 0.049999626016769494, "target_actual_norm": 9.231122970581055, "match_error": -2.6702880859375e-05, "scalar": 9.230277836031746, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.920141859251718, "first_log_odds": -17.75, "later_log_odds": -2.170141859251718, "first_differing_log_odds": -17.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-random1.npz", "array_sha256": "a893204027cfaf7f6d9c8cc56c5d8b0a406864dd429cc90fd66bef5969b0bca2", "seconds": 0.33784629218280315, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-random2", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "MXP<|im_end|>", "output_text": "MXP", "output_ids": [44, 27375, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.219045639038086, "actual_norm": 9.231103897094727, "relative_norm": 0.04999966734088336, "target_actual_norm": 9.231122970581055, "match_error": -1.9073486328125e-05, "scalar": 9.219044589313853, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 565, "cached": true, "scoring": {"normalized": "MXP", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-1.2379268584145353, -7.54708744774744e-08], "logp": -1.2379269338854098, "cache_id": 566, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.219045639038086, "actual_norm": 9.231103897094727, "relative_norm": 0.04999966734088336, "target_actual_norm": 9.231122970581055, "match_error": -1.9073486328125e-05, "scalar": 9.219044589313853, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-19.050426858414536, -1.9521509019366126], "logp": -21.002577760351148, "cache_id": 567, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 184.62330627441406, "strength_one_norm": 48.13352966308594, "target_shrink_scale": 0.19169541446361588, "requested_norm": 9.219045639038086, "actual_norm": 9.231103897094727, "relative_norm": 0.04999966734088336, "target_actual_norm": 9.231122970581055, "match_error": -1.9073486328125e-05, "scalar": 9.219044589313853, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.76465082646574, "first_log_odds": -17.8125, "later_log_odds": -1.95215082646574, "first_differing_log_odds": -17.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-random2.npz", "array_sha256": "48917066daadfff5b3ada86e87321ed1501dda7c22e3063d7110e8b37f2c6212", "seconds": 0.34748512506484985, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-main-bridge", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUR<|im_end|>", "output_text": "RUR", "output_ids": [49, 1511, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 102.04373168945312, 553.9043579101562, 861.863525390625, 84.60324096679688, 9.43889045715332, 174.8471221923828, 271.4949035644531, 104.46033477783203, 3.907834053039551, 63.282222747802734, 61.31043243408203, 112.2780532836914, 16.38966178894043, 20.11978530883789, 53.27336883544922, 74.561279296875, 137.6422882080078, 56.30582809448242, 58.90263748168945, 257.7940368652344, 219.12548828125, 251.8115234375, 23.639389038085938, 584.1002807617188, 304.6878967285156, 596.1345825195312, 131.30789184570312, 1005.4014892578125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 102.08582305908203, 553.8955078125, 861.8336181640625, 84.6469955444336, 9.552519798278809, 174.8833465576172, 271.4169921875, 104.50491333007812, 4.099088191986084, 63.27605438232422, 61.34966278076172, 112.25965118408203, 16.39571189880371, 20.120712280273438, 53.27833557128906, 74.541015625, 137.7470703125, 56.30297088623047, 58.95600891113281, 257.82562255859375, 219.11251831054688, 251.82931518554688, 23.727516174316406, 584.1233520507812, 304.6940612792969, 596.14501953125, 131.3074493408203, 1005.433349609375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.08956726640462875, 0.41489890217781067, 0.8485798239707947, 0.08727072179317474, 0.009390829131007195, 0.16246742010116577, 0.26556652784347534, 0.10250446200370789, 0.003607015823945403, 0.06298985332250595, 0.058820199221372604, 0.1140962615609169, 0.0169978104531765, 0.02037324570119381, 0.05117498338222504, 0.07087787985801697, 0.11164116114377975, 0.05640346556901932, 0.06628722697496414, 0.2854900062084198, 0.26404818892478943, 0.25160273909568787, 0.025436632335186005, 0.5113629102706909, 0.2943829596042633, 0.6402860879898071, 0.1271144300699234, 0.9586036205291748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 568, "cached": true, "scoring": {"normalized": "RUR", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-36.91794502172382, -1.192975318426118e-07], "logp": -36.91794514102135, "cache_id": 569, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 102.04373168945312, 553.9043579101562, 861.863525390625, 84.60324096679688, 9.43889045715332, 174.8471221923828, 271.4949035644531, 104.46033477783203, 3.907834053039551, 63.282222747802734, 61.31043243408203, 112.2780532836914, 16.38966178894043, 20.11978530883789, 53.27336883544922, 74.561279296875, 137.6422882080078, 56.30582809448242, 58.90263748168945, 257.7940368652344, 219.12548828125, 251.8115234375, 23.639389038085938, 584.1002807617188, 304.6878967285156, 596.1345825195312, 131.30789184570312, 1005.4014892578125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 102.08582305908203, 553.8955078125, 861.8336181640625, 84.6469955444336, 9.552519798278809, 174.8833465576172, 271.4169921875, 104.50491333007812, 4.099088191986084, 63.27605438232422, 61.34966278076172, 112.25965118408203, 16.39571189880371, 20.120712280273438, 53.27833557128906, 74.541015625, 137.7470703125, 56.30297088623047, 58.95600891113281, 257.82562255859375, 219.11251831054688, 251.82931518554688, 23.727516174316406, 584.1233520507812, 304.6940612792969, 596.14501953125, 131.3074493408203, 1005.433349609375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.08956726640462875, 0.41489890217781067, 0.8485798239707947, 0.08727072179317474, 0.009390829131007195, 0.16246742010116577, 0.26556652784347534, 0.10250446200370789, 0.003607015823945403, 0.06298985332250595, 0.058820199221372604, 0.1140962615609169, 0.0169978104531765, 0.02037324570119381, 0.05117498338222504, 0.07087787985801697, 0.11164116114377975, 0.05640346556901932, 0.06628722697496414, 0.2854900062084198, 0.26404818892478943, 0.25160273909568787, 0.025436632335186005, 0.5113629102706909, 0.2943829596042633, 0.6402860879898071, 0.1271144300699234, 0.9586036205291748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.011695021723819391, -1.8222416760430447], "logp": -1.833936697766864, "cache_id": 570, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 43.60091018676758, 122.07563018798828, 150.2443084716797, 6.01480770111084, 23.158926010131836, 102.04373168945312, 553.9043579101562, 861.863525390625, 84.60324096679688, 9.43889045715332, 174.8471221923828, 271.4949035644531, 104.46033477783203, 3.907834053039551, 63.282222747802734, 61.31043243408203, 112.2780532836914, 16.38966178894043, 20.11978530883789, 53.27336883544922, 74.561279296875, 137.6422882080078, 56.30582809448242, 58.90263748168945, 257.7940368652344, 219.12548828125, 251.8115234375, 23.639389038085938, 584.1002807617188, 304.6878967285156, 596.1345825195312, 131.30789184570312, 1005.4014892578125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 43.62457275390625, 122.1300277709961, 150.26268005371094, 6.192512035369873, 23.23377799987793, 102.08582305908203, 553.8955078125, 861.8336181640625, 84.6469955444336, 9.552519798278809, 174.8833465576172, 271.4169921875, 104.50491333007812, 4.099088191986084, 63.27605438232422, 61.34966278076172, 112.25965118408203, 16.39571189880371, 20.120712280273438, 53.27833557128906, 74.541015625, 137.7470703125, 56.30297088623047, 58.95600891113281, 257.82562255859375, 219.11251831054688, 251.82931518554688, 23.727516174316406, 584.1233520507812, 304.6940612792969, 596.14501953125, 131.3074493408203, 1005.433349609375]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.04004582390189171, 0.10715460777282715, 0.12224666029214859, 0.0053002964705228806, 0.020473623648285866, 0.08956726640462875, 0.41489890217781067, 0.8485798239707947, 0.08727072179317474, 0.009390829131007195, 0.16246742010116577, 0.26556652784347534, 0.10250446200370789, 0.003607015823945403, 0.06298985332250595, 0.058820199221372604, 0.1140962615609169, 0.0169978104531765, 0.02037324570119381, 0.05117498338222504, 0.07087787985801697, 0.11164116114377975, 0.05640346556901932, 0.06628722697496414, 0.2854900062084198, 0.26404818892478943, 0.25160273909568787, 0.025436632335186005, 0.5113629102706909, 0.2943829596042633, 0.6402860879898071, 0.1271144300699234, 0.9586036205291748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 35.08400844325448, "first_log_odds": 36.90625, "later_log_odds": -1.8222415567455172, "first_differing_log_odds": 36.90625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-main-bridge.npz", "array_sha256": "6c9801df8bf6c0a9957c33c5ff5b03ea78e048ef84272617b9579cca25420b2c", "seconds": 0.337160374969244, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-baseline", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 571, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.62500018557612, -5.590355070563422e-07], "logp": -22.625000744611626, "cache_id": 572, "trace": []}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.81250018557612, -2.0475826965605797], "logp": -33.8600828821367, "cache_id": 573, "trace": []}, "log_odds": -11.235082137525076, "first_log_odds": -9.1875, "later_log_odds": -2.047582137525076, "first_differing_log_odds": -9.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-baseline.npz", "array_sha256": "eee3b5d69f2ac9b89a8ed9dbe444ddbfd4d9f04a93753c1b28f36f8771c0cb59", "seconds": 1.2700588749721646, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-noop", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.278306007385254, "match_error": -9.278306007385254, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 574, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.62500018557612, -5.590355070563422e-07], "logp": -22.625000744611626, "cache_id": 575, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.278306007385254, "match_error": -9.278306007385254, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.81250018557612, -2.0475826965605797], "logp": -33.8600828821367, "cache_id": 576, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.278306007385254, "match_error": -9.278306007385254, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.235082137525076, "first_log_odds": -9.1875, "later_log_odds": -2.047582137525076, "first_differing_log_odds": -9.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-noop.npz", "array_sha256": "5309c69263eebf3328ff3c19e408cb94a63b200776c88bc301634cd006b0525a", "seconds": 1.2358411657623947, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-j", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.26611328125, "actual_norm": 9.278306007385254, "relative_norm": 0.04999489517083778, "target_actual_norm": 9.278306007385254, "match_error": 0.0, "scalar": 0.18997984378260852, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 577, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.25000021044427, -3.8862580396217154e-07], "logp": -23.250000599070074, "cache_id": 578, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.26611328125, "actual_norm": 9.278306007385254, "relative_norm": 0.04999489517083778, "target_actual_norm": 9.278306007385254, "match_error": 0.0, "scalar": 0.18997984378260852, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.00000021044427, -1.6698307747424497], "logp": -32.66983098518672, "cache_id": 579, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.26611328125, "actual_norm": 9.278306007385254, "relative_norm": 0.04999489517083778, "target_actual_norm": 9.278306007385254, "match_error": 0.0, "scalar": 0.18997984378260852, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.419830386116647, "first_log_odds": -7.75, "later_log_odds": -1.6698303861166472, "first_differing_log_odds": -7.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-j.npz", "array_sha256": "ec53e9f63e2c15b27bd2d0a33586dc6407e039b7a6c01a37443a7f3bebc6d48d", "seconds": 1.25745491636917, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-opposite", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Mexican peso, and its three-letter ISO 4217 code is **MXN**.<|im_end|>", "output_text": "The currency used in Puebla is the Mexican peso, and its three-letter ISO 4217 code is **MXN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 23279, 36935, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 29423, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.261176109313965, "actual_norm": 9.278087615966797, "relative_norm": 0.04999371839826062, "target_actual_norm": 9.278306007385254, "match_error": -0.00021839141845703125, "scalar": 9.261175345964148, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 580, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Mexican peso, and its three-letter ISO 4217 code is **MXN**.", "bare_format": false, "malformed": true, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Mexican peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Mexican", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.62500019655838, -6.680025684124689e-07], "logp": -22.625000864560946, "cache_id": 581, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.261176109313965, "actual_norm": 9.278087615966797, "relative_norm": 0.04999371839826062, "target_actual_norm": 9.278306007385254, "match_error": -0.00021839141845703125, "scalar": 9.261175345964148, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-32.250000196558375, -2.053457317689046], "logp": -34.30345751424742, "cache_id": 582, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.261176109313965, "actual_norm": 9.278087615966797, "relative_norm": 0.04999371839826062, "target_actual_norm": 9.278306007385254, "match_error": -0.00021839141845703125, "scalar": 9.261175345964148, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.678456649686474, "first_log_odds": -9.624999999999996, "later_log_odds": -2.0534566496864777, "first_differing_log_odds": -9.624999999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-opposite.npz", "array_sha256": "0e68edcdf596dffca83de0b2c6cab6e117d46473df25f9f5c27e52b85cdc2dcc", "seconds": 1.263917624950409, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-unrelated", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **MXN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **MXN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 29423, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266416549682617, "actual_norm": 9.278397560119629, "relative_norm": 0.049995388490346106, "target_actual_norm": 9.278306007385254, "match_error": 9.1552734375e-05, "scalar": 9.266418071631506, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 583, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **MXN**.", "bare_format": false, "malformed": true, "first_code": "MXN", "all_codes": ["MXN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.37500020154564, -4.391463951905686e-07], "logp": -22.375000640692033, "cache_id": 584, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266416549682617, "actual_norm": 9.278397560119629, "relative_norm": 0.049995388490346106, "target_actual_norm": 9.278306007385254, "match_error": 9.1552734375e-05, "scalar": 9.266418071631506, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.56250020154564, -1.968005687045781], "logp": -33.53050588859142, "cache_id": 585, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266416549682617, "actual_norm": 9.278397560119629, "relative_norm": 0.049995388490346106, "target_actual_norm": 9.278306007385254, "match_error": 9.1552734375e-05, "scalar": 9.266418071631506, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.155505247899388, "first_log_odds": -9.1875, "later_log_odds": -1.968005247899388, "first_differing_log_odds": -9.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-unrelated.npz", "array_sha256": "2d37307cca34c1419cd70e83aecd9904845ce23cbdae29b4e9def1db7baffb09", "seconds": 1.2473822496831417, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-ordinary", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.270674705505371, "actual_norm": 9.278640747070312, "relative_norm": 0.0499966988702901, "target_actual_norm": 9.278306007385254, "match_error": 0.00033473968505859375, "scalar": 9.270678615781435, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 586, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.875000172612744, -4.980654700665223e-07], "logp": -22.875000670678215, "cache_id": 587, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.270674705505371, "actual_norm": 9.278640747070312, "relative_norm": 0.0499966988702901, "target_actual_norm": 9.278306007385254, "match_error": 0.00033473968505859375, "scalar": 9.270678615781435, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.437500172612744, -2.081080515628112], "logp": -33.51858068824085, "cache_id": 588, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.270674705505371, "actual_norm": 9.278640747070312, "relative_norm": 0.0499966988702901, "target_actual_norm": 9.278306007385254, "match_error": 0.00033473968505859375, "scalar": 9.270678615781435, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.643580017562638, "first_log_odds": -8.5625, "later_log_odds": -2.0810800175626376, "first_differing_log_odds": -8.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-ordinary.npz", "array_sha256": "63235bfeea5816acc0f102175dede170cd5046aa3d11ae525f40b49ba774cb2a", "seconds": 1.2588657499291003, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-random0", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.2708740234375, "actual_norm": 9.278339385986328, "relative_norm": 0.049995075026908524, "target_actual_norm": 9.278306007385254, "match_error": 3.337860107421875e-05, "scalar": 9.270873282388493, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 589, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.625000164074475, -4.1521538979984044e-07], "logp": -22.625000579289864, "cache_id": 590, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.2708740234375, "actual_norm": 9.278339385986328, "relative_norm": 0.049995075026908524, "target_actual_norm": 9.278306007385254, "match_error": 3.337860107421875e-05, "scalar": 9.270873282388493, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.750000164074475, -1.9153647212912335], "logp": -33.66536488536571, "cache_id": 591, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.2708740234375, "actual_norm": 9.278339385986328, "relative_norm": 0.049995075026908524, "target_actual_norm": 9.278306007385254, "match_error": 3.337860107421875e-05, "scalar": 9.270873282388493, "random_seed": 20271908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.040364306075848, "first_log_odds": -9.125, "later_log_odds": -1.9153643060758476, "first_differing_log_odds": -9.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-random0.npz", "array_sha256": "b62a87e95dba24567a05f674359a7fc7195bd239d78f704875348e1927eb46e0", "seconds": 1.248045583255589, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-random1", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.27466106414795, "actual_norm": 9.278525352478027, "relative_norm": 0.049996077082159814, "target_actual_norm": 9.278306007385254, "match_error": 0.0002193450927734375, "scalar": 9.274660432743985, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 592, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-22.50000019831512, -5.543732874879341e-07], "logp": -22.500000752688408, "cache_id": 593, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.27466106414795, "actual_norm": 9.278525352478027, "relative_norm": 0.049996077082159814, "target_actual_norm": 9.278306007385254, "match_error": 0.0002193450927734375, "scalar": 9.274660432743985, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.81250019831512, -1.9969979833163887], "logp": -33.80949818163151, "cache_id": 594, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.27466106414795, "actual_norm": 9.278525352478027, "relative_norm": 0.049996077082159814, "target_actual_norm": 9.278306007385254, "match_error": 0.0002193450927734375, "scalar": 9.274660432743985, "random_seed": 20271909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.309497428943104, "first_log_odds": -9.3125, "later_log_odds": -1.9969974289431036, "first_differing_log_odds": -9.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-random1.npz", "array_sha256": "3866388f02cfd81a64fb0b5af34204441560053a93feb00a28e97d456afba6b6", "seconds": 1.2645440003834665, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-random2", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 393, 28830, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266910552978516, "actual_norm": 9.278470993041992, "relative_norm": 0.04999578417370175, "target_actual_norm": 9.278306007385254, "match_error": 0.00016498565673828125, "scalar": 9.26690916239022, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 595, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the Peso, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-23.125000149582746, -5.445223239710557e-07], "logp": -23.12500069410507, "cache_id": 596, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266910552978516, "actual_norm": 9.278470993041992, "relative_norm": 0.04999578417370175, "target_actual_norm": 9.278306007385254, "match_error": 0.00016498565673828125, "scalar": 9.26690916239022, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-31.937500149582746, -1.9738327918945506], "logp": -33.9113329414773, "cache_id": 597, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 185.58506774902344, "strength_one_norm": 48.77418899536133, "target_shrink_scale": 0.18997984378260852, "requested_norm": 9.266910552978516, "actual_norm": 9.278470993041992, "relative_norm": 0.04999578417370175, "target_actual_norm": 9.278306007385254, "match_error": 0.00016498565673828125, "scalar": 9.26690916239022, "random_seed": 20271910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.78633224737223, "first_log_odds": -8.8125, "later_log_odds": -1.9738322473722292, "first_differing_log_odds": -8.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-random2.npz", "array_sha256": "ecc8f39a4027aa45a879ad71ebb6faa674322a716bcce33e7be6b1476cee9f6f", "seconds": 1.2446638327091932, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-10-paraphrase-bridge", "case_id": "eval-10", "city": "Puebla", "country": "Mexico", "code": "MXN", "target_country": "Russia", "target_code": "RUB", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Puebla is the **Peso**, and its three-letter ISO 4217 code is **PEN**.<|im_end|>", "output_text": "The currency used in Puebla is the **Peso**, and its three-letter ISO 4217 code is **PEN**.", "output_ids": [785, 11413, 1483, 304, 393, 361, 64726, 374, 279, 3070, 47, 28830, 97219, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 47, 953, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 92.25723266601562, 608.6470947265625, 878.849365234375, 218.23179626464844, 109.44677734375, 333.62396240234375, 243.84368896484375, 97.60359191894531, 13.227811813354492, 53.08356475830078, 61.31755065917969, 101.11198425292969, 26.522397994995117, 25.711671829223633, 20.178163528442383, 104.67855072021484, 226.1322784423828, 252.84193420410156, 279.56622314453125, 18.57095718383789, 473.9001770019531, 276.078369140625, 516.6209716796875, 142.95460510253906, 741.9512939453125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 92.20088958740234, 608.5768432617188, 878.9253540039062, 218.2310333251953, 109.40019989013672, 333.5828552246094, 243.82803344726562, 97.63602447509766, 13.26474380493164, 53.17075729370117, 61.34968948364258, 101.0987319946289, 26.58521842956543, 25.689199447631836, 20.209135055541992, 104.68665313720703, 226.20147705078125, 252.83489990234375, 279.5740051269531, 18.611425399780273, 473.8997802734375, 276.0562438964844, 516.6315307617188, 142.96434020996094, 741.8203125]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.08243532478809357, 0.46947070956230164, 0.8529601693153381, 0.2187674194574356, 0.10094275325536728, 0.3131745457649231, 0.22620832920074463, 0.09607444703578949, 0.01140270009636879, 0.051066625863313675, 0.058106862008571625, 0.10263446718454361, 0.02733004465699196, 0.025568092241883278, 0.0191607978194952, 0.10982728004455566, 0.25456222891807556, 0.2921121120452881, 0.2728322446346283, 0.02001151442527771, 0.41643503308296204, 0.26221323013305664, 0.5441363453865051, 0.13807255029678345, 0.7797580361366272]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 598, "cached": true, "scoring": {"normalized": "The currency used in Puebla is the **Peso**, and its three-letter ISO 4217 code is **PEN**.", "bare_format": false, "malformed": true, "first_code": "PEN", "all_codes": ["PEN"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Puebla is the **Peso**, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [29423, 45], "tokens": ["MX", "N"], "log_probs": [-42.442202290716494, -8.984551667751832e-07], "logp": -42.442203189171664, "cache_id": 599, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 92.25723266601562, 608.6470947265625, 878.849365234375, 218.23179626464844, 109.44677734375, 333.62396240234375, 243.84368896484375, 97.60359191894531, 13.227811813354492, 53.08356475830078, 61.31755065917969, 101.11198425292969, 26.522397994995117, 25.711671829223633, 20.178163528442383, 104.67855072021484, 226.1322784423828, 252.84193420410156, 279.56622314453125, 18.57095718383789, 473.9001770019531, 276.078369140625, 516.6209716796875, 142.95460510253906, 741.9512939453125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 92.20088958740234, 608.5768432617188, 878.9253540039062, 218.2310333251953, 109.40019989013672, 333.5828552246094, 243.82803344726562, 97.63602447509766, 13.26474380493164, 53.17075729370117, 61.34968948364258, 101.0987319946289, 26.58521842956543, 25.689199447631836, 20.209135055541992, 104.68665313720703, 226.20147705078125, 252.83489990234375, 279.5740051269531, 18.611425399780273, 473.8997802734375, 276.0562438964844, 516.6315307617188, 142.96434020996094, 741.8203125]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.08243532478809357, 0.46947070956230164, 0.8529601693153381, 0.2187674194574356, 0.10094275325536728, 0.3131745457649231, 0.22620832920074463, 0.09607444703578949, 0.01140270009636879, 0.051066625863313675, 0.058106862008571625, 0.10263446718454361, 0.02733004465699196, 0.025568092241883278, 0.0191607978194952, 0.10982728004455566, 0.25456222891807556, 0.2921121120452881, 0.2728322446346283, 0.02001151442527771, 0.41643503308296204, 0.26221323013305664, 0.5441363453865051, 0.13807255029678345, 0.7797580361366272]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-6.1297022907164935, -2.1476726383290714], "logp": -8.277374929045564, "cache_id": 600, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[429.12396240234375, 93.662353515625, 6.42195987701416, 87.17338562011719, 55.555450439453125, 128.67892456054688, 153.33058166503906, 3.927245616912842, 92.25723266601562, 608.6470947265625, 878.849365234375, 218.23179626464844, 109.44677734375, 333.62396240234375, 243.84368896484375, 97.60359191894531, 13.227811813354492, 53.08356475830078, 61.31755065917969, 101.11198425292969, 26.522397994995117, 25.711671829223633, 20.178163528442383, 104.67855072021484, 226.1322784423828, 252.84193420410156, 279.56622314453125, 18.57095718383789, 473.9001770019531, 276.078369140625, 516.6209716796875, 142.95460510253906, 741.9512939453125]], "actual_norm": [[429.1748046875, 93.66996765136719, 6.512460708618164, 87.15390014648438, 55.56703567504883, 128.71527099609375, 153.33950805664062, 4.061587810516357, 92.20088958740234, 608.5768432617188, 878.9253540039062, 218.2310333251953, 109.40019989013672, 333.5828552246094, 243.82803344726562, 97.63602447509766, 13.26474380493164, 53.17075729370117, 61.34968948364258, 101.0987319946289, 26.58521842956543, 25.689199447631836, 20.209135055541992, 104.68665313720703, 226.20147705078125, 252.83489990234375, 279.5740051269531, 18.611425399780273, 473.8997802734375, 276.0562438964844, 516.6315307617188, 142.96434020996094, 741.8203125]], "relative_norm": [[0.02504873275756836, 0.09936311841011047, 0.007204811554402113, 0.07598908990621567, 0.05058129504323006, 0.11181958019733429, 0.1363784223794937, 0.0036079336423426867, 0.08243532478809357, 0.46947070956230164, 0.8529601693153381, 0.2187674194574356, 0.10094275325536728, 0.3131745457649231, 0.22620832920074463, 0.09607444703578949, 0.01140270009636879, 0.051066625863313675, 0.058106862008571625, 0.10263446718454361, 0.02733004465699196, 0.025568092241883278, 0.0191607978194952, 0.10982728004455566, 0.25456222891807556, 0.2921121120452881, 0.2728322446346283, 0.02001151442527771, 0.41643503308296204, 0.26221323013305664, 0.5441363453865051, 0.13807255029678345, 0.7797580361366272]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 34.1648282601261, "first_log_odds": 36.3125, "later_log_odds": -2.1476717398739, "first_differing_log_odds": 36.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-10-paraphrase-bridge.npz", "array_sha256": "a6b2d6cc9ecc9fb0f04d6d34955af7f8671209f18b21b2626b2d6478c3e2b80a", "seconds": 1.2866554167121649, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-baseline", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 601, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.006729947620533484, -0.06702841741314053], "logp": -0.07375836503367401, "cache_id": 602, "trace": []}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.694229947620535], "logp": -20.694229947620535, "cache_id": 603, "trace": []}, "log_odds": -20.620471582586863, "first_log_odds": -20.6875, "later_log_odds": 0.06702841741313748, "first_differing_log_odds": -20.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-baseline.npz", "array_sha256": "2a1cb8c4ecb736c5c28ed9e2ee16b7936309d3c30a98f94006861441574ecf10", "seconds": 0.35079670790582895, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456879104, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-noop", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.191278457641602, "match_error": -9.191278457641602, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 604, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.006729947620533484, -0.06702841741314053], "logp": -0.07375836503367401, "cache_id": 605, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.191278457641602, "match_error": -9.191278457641602, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.694229947620535], "logp": -20.694229947620535, "cache_id": 606, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.191278457641602, "match_error": -9.191278457641602, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.620471582586863, "first_log_odds": -20.6875, "later_log_odds": 0.06702841741313748, "first_differing_log_odds": -20.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-noop.npz", "array_sha256": "82cc0d58cc64263c474c484e134c91a81e8d704228f9cda79d8087ec98507d15", "seconds": 0.3227682081051171, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-j", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.192042350769043, "actual_norm": 9.191278457641602, "relative_norm": 0.04999585207473453, "target_actual_norm": 9.191278457641602, "match_error": 0.0, "scalar": 0.21900654991472004, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 607, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.010409936428576795, -0.05723047771344355], "logp": -0.06764041414202035, "cache_id": 608, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.192042350769043, "actual_norm": 9.191278457641602, "relative_norm": 0.04999585207473453, "target_actual_norm": 9.191278457641602, "match_error": 0.0, "scalar": 0.21900654991472004, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.072909936428577], "logp": -20.072909936428577, "cache_id": 609, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.192042350769043, "actual_norm": 9.191278457641602, "relative_norm": 0.04999585207473453, "target_actual_norm": 9.191278457641602, "match_error": 0.0, "scalar": 0.21900654991472004, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.005269522286557, "first_log_odds": -20.0625, "later_log_odds": 0.057230477713442696, "first_differing_log_odds": -20.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-j.npz", "array_sha256": "c889ffc12659ee71bfcd2dd657dcbbf61eda9bca3e0c010eb35df16a1bd1c3c0", "seconds": 0.31405574968084693, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-opposite", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.20467472076416, "actual_norm": 9.191238403320312, "relative_norm": 0.04999563419972059, "target_actual_norm": 9.191278457641602, "match_error": -4.00543212890625e-05, "scalar": 9.204672120555188, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 610, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.007688965522282787, -0.08059012803747126], "logp": -0.08827909355975405, "cache_id": 611, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.20467472076416, "actual_norm": 9.191238403320312, "relative_norm": 0.04999563419972059, "target_actual_norm": 9.191278457641602, "match_error": -4.00543212890625e-05, "scalar": 9.204672120555188, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.757688965522284], "logp": -20.757688965522284, "cache_id": 612, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.20467472076416, "actual_norm": 9.191238403320312, "relative_norm": 0.04999563419972059, "target_actual_norm": 9.191278457641602, "match_error": -4.00543212890625e-05, "scalar": 9.204672120555188, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.66940987196253, "first_log_odds": -20.75, "later_log_odds": 0.08059012803747123, "first_differing_log_odds": -20.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-opposite.npz", "array_sha256": "ab2f39e2f087447a03eab69597922fc3c763db0da95dc23048e955f9e6c32a2b", "seconds": 0.31890579080209136, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-unrelated", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.202499389648438, "actual_norm": 9.1912260055542, "relative_norm": 0.049995566762216274, "target_actual_norm": 9.191278457641602, "match_error": -5.245208740234375e-05, "scalar": 9.20249827997759, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 613, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.010870990836097588, -0.055142323425373896], "logp": -0.06601331426147149, "cache_id": 614, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.202499389648438, "actual_norm": 9.1912260055542, "relative_norm": 0.049995566762216274, "target_actual_norm": 9.191278457641602, "match_error": -5.245208740234375e-05, "scalar": 9.20249827997759, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.260870990836096], "logp": -20.260870990836096, "cache_id": 615, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.202499389648438, "actual_norm": 9.1912260055542, "relative_norm": 0.049995566762216274, "target_actual_norm": 9.191278457641602, "match_error": -5.245208740234375e-05, "scalar": 9.20249827997759, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.194857676574625, "first_log_odds": -20.25, "later_log_odds": 0.055142323425375395, "first_differing_log_odds": -20.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-unrelated.npz", "array_sha256": "8f6ffdf3cb94155658e17b68bbfceaf075764b55ec887cadcf60f54892f2aa10", "seconds": 0.3166122091934085, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456911872, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-ordinary", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.173641204833984, "actual_norm": 9.191309928894043, "relative_norm": 0.04999602326224549, "target_actual_norm": 9.191278457641602, "match_error": 3.147125244140625e-05, "scalar": 9.17364229940722, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 616, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.008480924634516234, -0.08183310962163484], "logp": -0.09031403425615107, "cache_id": 617, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.173641204833984, "actual_norm": 9.191309928894043, "relative_norm": 0.04999602326224549, "target_actual_norm": 9.191278457641602, "match_error": 3.147125244140625e-05, "scalar": 9.17364229940722, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.508480924634515], "logp": -20.508480924634515, "cache_id": 618, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.173641204833984, "actual_norm": 9.191309928894043, "relative_norm": 0.04999602326224549, "target_actual_norm": 9.191278457641602, "match_error": 3.147125244140625e-05, "scalar": 9.17364229940722, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.418166890378362, "first_log_odds": -20.5, "later_log_odds": 0.08183310962163759, "first_differing_log_odds": -20.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-ordinary.npz", "array_sha256": "fb78c34aef88da6a8a090a54172052168074918e20e6b5dc48ac3dedf4f2d490", "seconds": 0.31765862461179495, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456944640, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-random0", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.197643280029297, "actual_norm": 9.191322326660156, "relative_norm": 0.04999609069974981, "target_actual_norm": 9.191278457641602, "match_error": 4.38690185546875e-05, "scalar": 9.197642200622795, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 619, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.006669574378427211, -0.05279203934122005], "logp": -0.05946161371964726, "cache_id": 620, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.197643280029297, "actual_norm": 9.191322326660156, "relative_norm": 0.04999609069974981, "target_actual_norm": 9.191278457641602, "match_error": 4.38690185546875e-05, "scalar": 9.197642200622795, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.631669574378428], "logp": -20.631669574378428, "cache_id": 621, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.197643280029297, "actual_norm": 9.191322326660156, "relative_norm": 0.04999609069974981, "target_actual_norm": 9.191278457641602, "match_error": 4.38690185546875e-05, "scalar": 9.197642200622795, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.57220796065878, "first_log_odds": -20.625, "later_log_odds": 0.05279203934122023, "first_differing_log_odds": -20.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-random0.npz", "array_sha256": "4a3810a05395a319d75f37a26d23dde691dc7147d8ffd3fa5a2f8670e790faed", "seconds": 0.31490708282217383, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456944640, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-random1", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.189314842224121, "actual_norm": 9.191276550292969, "relative_norm": 0.04999584169973387, "target_actual_norm": 9.191278457641602, "match_error": -1.9073486328125e-06, "scalar": 9.189314988732804, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 622, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.007661395380840787, -0.06421974542425823], "logp": -0.07188114080509903, "cache_id": 623, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.189314842224121, "actual_norm": 9.191276550292969, "relative_norm": 0.04999584169973387, "target_actual_norm": 9.191278457641602, "match_error": -1.9073486328125e-06, "scalar": 9.189314988732804, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.50766139538084], "logp": -20.50766139538084, "cache_id": 624, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.189314842224121, "actual_norm": 9.191276550292969, "relative_norm": 0.04999584169973387, "target_actual_norm": 9.191278457641602, "match_error": -1.9073486328125e-06, "scalar": 9.189314988732804, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.43578025457574, "first_log_odds": -20.5, "later_log_odds": 0.06421974542426057, "first_differing_log_odds": -20.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-random1.npz", "array_sha256": "0634f8afd4b947518fa997933efb7d42705a79877810ded90c422f51c943a1bd", "seconds": 0.316638124641031, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456944640, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-random2", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.179496765136719, "actual_norm": 9.191288948059082, "relative_norm": 0.049995909137238184, "target_actual_norm": 9.191278457641602, "match_error": 1.049041748046875e-05, "scalar": 9.179497644188814, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 625, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.00839240503979358, -0.06230793920950643], "logp": -0.07070034424930001, "cache_id": 626, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.179496765136719, "actual_norm": 9.191288948059082, "relative_norm": 0.049995909137238184, "target_actual_norm": 9.191278457641602, "match_error": 1.049041748046875e-05, "scalar": 9.179497644188814, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.820892405039793], "logp": -20.820892405039793, "cache_id": 627, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 183.8408203125, "strength_one_norm": 41.971534729003906, "target_shrink_scale": 0.21900654991472004, "requested_norm": 9.179496765136719, "actual_norm": 9.191288948059082, "relative_norm": 0.049995909137238184, "target_actual_norm": 9.191278457641602, "match_error": 1.049041748046875e-05, "scalar": 9.179497644188814, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -20.750192060790493, "first_log_odds": -20.8125, "later_log_odds": 0.06230793920950717, "first_differing_log_odds": -20.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-random2.npz", "array_sha256": "b1ce1cc83b9bfdfd80812c86f5d7aa3a1553d1867d7c628da032aff9b36cda92", "seconds": 0.3163038333877921, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456944640, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-main-bridge", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 527.6719360351562, 51.454864501953125, 109.41385650634766, 4.793062210083008, 18.944028854370117, 28.75313377380371, 55.818397521972656, 46.68067932128906, 21.12343406677246, 183.1407012939453, 161.46664428710938, 42.43776321411133, 217.17958068847656, 9.839336395263672, 45.50764083862305, 46.204837799072266, 26.404296875, 160.33445739746094, 159.1433868408203, 410.1996154785156, 141.7568817138672, 329.8291320800781, 356.5368347167969, 530.4025268554688, 181.76495361328125, 610.5151977539062]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 527.6680908203125, 51.45831298828125, 109.42112731933594, 4.9214186668396, 18.977449417114258, 28.772375106811523, 55.80064010620117, 46.689247131347656, 21.14779281616211, 183.17335510253906, 161.4100799560547, 42.38534927368164, 217.1517333984375, 9.927541732788086, 45.51899719238281, 46.23838424682617, 26.439922332763672, 160.25906372070312, 159.0956573486328, 410.2220458984375, 141.78297424316406, 329.7718811035156, 356.54766845703125, 530.45947265625, 181.76467895507812, 610.5322265625]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.49415281414985657, 0.05289187654852867, 0.10784544795751572, 0.004600002896040678, 0.018435882404446602, 0.027862243354320526, 0.0486486479640007, 0.04545033723115921, 0.02025788091123104, 0.18645818531513214, 0.16540087759494781, 0.04235420748591423, 0.20970280468463898, 0.00942390225827694, 0.036703791469335556, 0.04679443687200546, 0.029586883261799812, 0.1770170032978058, 0.1901426762342453, 0.4057987630367279, 0.15102970600128174, 0.2916930913925171, 0.3438819348812103, 0.5719566345214844, 0.17517900466918945, 0.5527101159095764]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}], "cache_id": 628, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-11.570480521891621, -0.030865836903131057], "logp": -11.601346358794752, "cache_id": 629, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 527.6719360351562, 51.454864501953125, 109.41385650634766, 4.793062210083008, 18.944028854370117, 28.75313377380371, 55.818397521972656, 46.68067932128906, 21.12343406677246, 183.1407012939453, 161.46664428710938, 42.43776321411133, 217.17958068847656, 9.839336395263672, 45.50764083862305, 46.204837799072266, 26.404296875, 160.33445739746094, 159.1433868408203, 410.1996154785156, 141.7568817138672, 329.8291320800781, 356.5368347167969, 530.4025268554688, 181.76495361328125, 610.5151977539062]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 527.6680908203125, 51.45831298828125, 109.42112731933594, 4.9214186668396, 18.977449417114258, 28.772375106811523, 55.80064010620117, 46.689247131347656, 21.14779281616211, 183.17335510253906, 161.4100799560547, 42.38534927368164, 217.1517333984375, 9.927541732788086, 45.51899719238281, 46.23838424682617, 26.439922332763672, 160.25906372070312, 159.0956573486328, 410.2220458984375, 141.78297424316406, 329.7718811035156, 356.54766845703125, 530.45947265625, 181.76467895507812, 610.5322265625]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.49415281414985657, 0.05289187654852867, 0.10784544795751572, 0.004600002896040678, 0.018435882404446602, 0.027862243354320526, 0.0486486479640007, 0.04545033723115921, 0.02025788091123104, 0.18645818531513214, 0.16540087759494781, 0.04235420748591423, 0.20970280468463898, 0.00942390225827694, 0.036703791469335556, 0.04679443687200546, 0.029586883261799812, 0.1770170032978058, 0.1901426762342453, 0.4057987630367279, 0.15102970600128174, 0.2916930913925171, 0.3438819348812103, 0.5719566345214844, 0.17517900466918945, 0.5527101159095764]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-5.6954805218916205], "logp": -5.6954805218916205, "cache_id": 630, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 527.6719360351562, 51.454864501953125, 109.41385650634766, 4.793062210083008, 18.944028854370117, 28.75313377380371, 55.818397521972656, 46.68067932128906, 21.12343406677246, 183.1407012939453, 161.46664428710938, 42.43776321411133, 217.17958068847656, 9.839336395263672, 45.50764083862305, 46.204837799072266, 26.404296875, 160.33445739746094, 159.1433868408203, 410.1996154785156, 141.7568817138672, 329.8291320800781, 356.5368347167969, 530.4025268554688, 181.76495361328125, 610.5151977539062]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 527.6680908203125, 51.45831298828125, 109.42112731933594, 4.9214186668396, 18.977449417114258, 28.772375106811523, 55.80064010620117, 46.689247131347656, 21.14779281616211, 183.17335510253906, 161.4100799560547, 42.38534927368164, 217.1517333984375, 9.927541732788086, 45.51899719238281, 46.23838424682617, 26.439922332763672, 160.25906372070312, 159.0956573486328, 410.2220458984375, 141.78297424316406, 329.7718811035156, 356.54766845703125, 530.45947265625, 181.76467895507812, 610.5322265625]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.49415281414985657, 0.05289187654852867, 0.10784544795751572, 0.004600002896040678, 0.018435882404446602, 0.027862243354320526, 0.0486486479640007, 0.04545033723115921, 0.02025788091123104, 0.18645818531513214, 0.16540087759494781, 0.04235420748591423, 0.20970280468463898, 0.00942390225827694, 0.036703791469335556, 0.04679443687200546, 0.029586883261799812, 0.1770170032978058, 0.1901426762342453, 0.4057987630367279, 0.15102970600128174, 0.2916930913925171, 0.3438819348812103, 0.5719566345214844, 0.17517900466918945, 0.5527101159095764]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "log_odds": 5.905865836903131, "first_log_odds": 5.875000000000001, "later_log_odds": 0.030865836903130273, "first_differing_log_odds": 5.875000000000001, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-main-bridge.npz", "array_sha256": "4c961bfcae41bdef6afa44b50b742cf89a8df323f340480bc24f835ac55951b3", "seconds": 0.2876637917943299, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-baseline", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 631, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.375005167384712, -0.03815361956350406], "logp": -14.413158786948216, "cache_id": 632, "trace": []}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.062505167384714], "logp": -31.062505167384714, "cache_id": 633, "trace": []}, "log_odds": -16.649346380436498, "first_log_odds": -16.6875, "later_log_odds": 0.03815361956350216, "first_differing_log_odds": -16.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-baseline.npz", "array_sha256": "6436116c22b2fe652124e700288f8cbc903f910c14f83cf8bf80435aded84e65", "seconds": 1.1529482500627637, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-noop", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.122419357299805, "match_error": -9.122419357299805, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 634, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.375005167384712, -0.03815361956350406], "logp": -14.413158786948216, "cache_id": 635, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.122419357299805, "match_error": -9.122419357299805, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.062505167384714], "logp": -31.062505167384714, "cache_id": 636, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.122419357299805, "match_error": -9.122419357299805, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.649346380436498, "first_log_odds": -16.6875, "later_log_odds": 0.03815361956350216, "first_differing_log_odds": -16.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-noop.npz", "array_sha256": "09d660ec818e127d9c832a9689d43543cad51de389ba8fbae2a91f95ba2bc0c0", "seconds": 1.1704638749361038, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-j", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.12505054473877, "actual_norm": 9.122419357299805, "relative_norm": 0.04998558783851858, "target_actual_norm": 9.122419357299805, "match_error": 0.0, "scalar": 0.2058053833754696, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 637, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.8750042748298, -0.035753078654503376], "logp": -14.910757353484303, "cache_id": 638, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.12505054473877, "actual_norm": 9.122419357299805, "relative_norm": 0.04998558783851858, "target_actual_norm": 9.122419357299805, "match_error": 0.0, "scalar": 0.2058053833754696, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-30.8125042748298], "logp": -30.8125042748298, "cache_id": 639, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.12505054473877, "actual_norm": 9.122419357299805, "relative_norm": 0.04998558783851858, "target_actual_norm": 9.122419357299805, "match_error": 0.0, "scalar": 0.2058053833754696, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -15.901746921345495, "first_log_odds": -15.937499999999998, "later_log_odds": 0.03575307865450306, "first_differing_log_odds": -15.937499999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-j.npz", "array_sha256": "17fb3f841a91fdacc7321d07d60fe76e9edb8f9ad4e852bbaa14ad1424e550a0", "seconds": 1.1617782921530306, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-opposite", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.129205703735352, "actual_norm": 9.12231159210205, "relative_norm": 0.04998499734749341, "target_actual_norm": 9.122419357299805, "match_error": -0.00010776519775390625, "scalar": 9.129205214594549, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 640, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.750004865075729, -0.031009602748429628], "logp": -14.781014467824159, "cache_id": 641, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.129205703735352, "actual_norm": 9.12231159210205, "relative_norm": 0.04998499734749341, "target_actual_norm": 9.122419357299805, "match_error": -0.00010776519775390625, "scalar": 9.129205214594549, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.43750486507573], "logp": -31.43750486507573, "cache_id": 642, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.129205703735352, "actual_norm": 9.12231159210205, "relative_norm": 0.04998499734749341, "target_actual_norm": 9.122419357299805, "match_error": -0.00010776519775390625, "scalar": 9.129205214594549, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.656490397251574, "first_log_odds": -16.6875, "later_log_odds": 0.031009602748426346, "first_differing_log_odds": -16.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-opposite.npz", "array_sha256": "b5a20c325931a3d2879b951104f2f681e00aae8ee4a14f0e3f0c043eab1b220f", "seconds": 1.1566807911731303, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-unrelated", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124543190002441, "actual_norm": 9.122340202331543, "relative_norm": 0.04998515411502222, "target_actual_norm": 9.122419357299805, "match_error": -7.915496826171875e-05, "scalar": 9.124542112658673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 643, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-15.37500327225619, -0.032460605267643244], "logp": -15.407463877523833, "cache_id": 644, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124543190002441, "actual_norm": 9.122340202331543, "relative_norm": 0.04998515411502222, "target_actual_norm": 9.122419357299805, "match_error": -7.915496826171875e-05, "scalar": 9.124542112658673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.50000327225619], "logp": -31.50000327225619, "cache_id": 645, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124543190002441, "actual_norm": 9.122340202331543, "relative_norm": 0.04998515411502222, "target_actual_norm": 9.122419357299805, "match_error": -7.915496826171875e-05, "scalar": 9.124542112658673, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.092539394732356, "first_log_odds": -16.125, "later_log_odds": 0.03246060526764438, "first_differing_log_odds": -16.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-unrelated.npz", "array_sha256": "c1f8d39f638feb1706ffce9a6d274792df31e8c03bebfb233e09b1bae5765fb9", "seconds": 1.1637971252202988, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-ordinary", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.113232612609863, "actual_norm": 9.12243366241455, "relative_norm": 0.04998566622228298, "target_actual_norm": 9.122419357299805, "match_error": 1.430511474609375e-05, "scalar": 9.113232350500766, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 646, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.625004935853127, -0.028708395779269386], "logp": -14.653713331632396, "cache_id": 647, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.113232612609863, "actual_norm": 9.12243366241455, "relative_norm": 0.04998566622228298, "target_actual_norm": 9.122419357299805, "match_error": 1.430511474609375e-05, "scalar": 9.113232350500766, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.062504935853127], "logp": -31.062504935853127, "cache_id": 648, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.113232612609863, "actual_norm": 9.12243366241455, "relative_norm": 0.04998566622228298, "target_actual_norm": 9.122419357299805, "match_error": 1.430511474609375e-05, "scalar": 9.113232350500766, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.40879160422073, "first_log_odds": -16.4375, "later_log_odds": 0.02870839577926887, "first_differing_log_odds": -16.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-ordinary.npz", "array_sha256": "96f8b00451160b6261fbee41627f107ffb9d6258b2bb7670b4faf702b038be70", "seconds": 1.16144883306697, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-random0", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.131397247314453, "actual_norm": 9.122138023376465, "relative_norm": 0.049984046291151996, "target_actual_norm": 9.122419357299805, "match_error": -0.00028133392333984375, "scalar": 9.131397568489774, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 649, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.50000550228796, -0.03239113652294402], "logp": -14.532396638810905, "cache_id": 650, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.131397247314453, "actual_norm": 9.122138023376465, "relative_norm": 0.049984046291151996, "target_actual_norm": 9.122419357299805, "match_error": -0.00028133392333984375, "scalar": 9.131397568489774, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.062505502287962], "logp": -31.062505502287962, "cache_id": 651, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.131397247314453, "actual_norm": 9.122138023376465, "relative_norm": 0.049984046291151996, "target_actual_norm": 9.122419357299805, "match_error": -0.00028133392333984375, "scalar": 9.131397568489774, "random_seed": 20272008, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.530108863477057, "first_log_odds": -16.5625, "later_log_odds": 0.03239113652294279, "first_differing_log_odds": -16.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-random0.npz", "array_sha256": "29295ab317bf6e0c519986847baa1b97eeb6b09170a83c0741ffbe00a1cda57b", "seconds": 1.1593922497704625, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-random1", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124056816101074, "actual_norm": 9.122438430786133, "relative_norm": 0.04998569235020445, "target_actual_norm": 9.122419357299805, "match_error": 1.9073486328125e-05, "scalar": 9.124054922904179, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 652, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.37500562251843, -0.03807860034111843], "logp": -14.413084222859547, "cache_id": 653, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124056816101074, "actual_norm": 9.122438430786133, "relative_norm": 0.04998569235020445, "target_actual_norm": 9.122419357299805, "match_error": 1.9073486328125e-05, "scalar": 9.124054922904179, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.00000562251843], "logp": -31.00000562251843, "cache_id": 654, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.124056816101074, "actual_norm": 9.122438430786133, "relative_norm": 0.04998569235020445, "target_actual_norm": 9.122419357299805, "match_error": 1.9073486328125e-05, "scalar": 9.124054922904179, "random_seed": 20272009, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.586921399658884, "first_log_odds": -16.625, "later_log_odds": 0.03807860034111599, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-random1.npz", "array_sha256": "1ae23d011c078caf82d75ee583ad667d7824f91e05fe0a611b28c52b5d0f3ed6", "seconds": 1.1524879578500986, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-random2", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.116571426391602, "actual_norm": 9.1224365234375, "relative_norm": 0.04998568189903586, "target_actual_norm": 9.122419357299805, "match_error": 1.71661376953125e-05, "scalar": 9.116573080245871, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 655, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-14.500006039337817, -0.038207235721308457], "logp": -14.538213275059126, "cache_id": 656, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.116571426391602, "actual_norm": 9.1224365234375, "relative_norm": 0.04998568189903586, "target_actual_norm": 9.122419357299805, "match_error": 1.71661376953125e-05, "scalar": 9.116573080245871, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.000006039337816], "logp": -31.000006039337816, "cache_id": 657, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 182.50099182128906, "strength_one_norm": 44.3382453918457, "target_shrink_scale": 0.2058053833754696, "requested_norm": 9.116571426391602, "actual_norm": 9.1224365234375, "relative_norm": 0.04998568189903586, "target_actual_norm": 9.122419357299805, "match_error": 1.71661376953125e-05, "scalar": 9.116573080245871, "random_seed": 20272010, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -16.46179276427869, "first_log_odds": -16.5, "later_log_odds": 0.03820723572130902, "first_differing_log_odds": -16.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-random2.npz", "array_sha256": "d52981186746e5bcf2f01b2a007ed3332cf22b503dee509c8004a4e6b65abc02", "seconds": 1.169124124571681, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-11-paraphrase-bridge", "case_id": "eval-11", "city": "Moscow", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 22415, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 575.3573608398438, 29.356595993041992, 10.482144355773926, 150.31640625, 61.959598541259766, 33.22575378417969, 73.96873474121094, 45.55012893676758, 42.7547607421875, 194.24264526367188, 153.03224182128906, 26.38029670715332, 177.20054626464844, 84.02604675292969, 150.96775817871094, 202.09344482421875, 426.6070251464844, 185.06240844726562, 296.5726623535156, 353.2792663574219, 584.0364990234375, 177.91868591308594, 345.28900146484375]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 575.3670043945312, 29.386768341064453, 10.622042655944824, 150.2969970703125, 61.987579345703125, 33.25994873046875, 73.98925018310547, 45.56269836425781, 42.82190704345703, 194.26107788085938, 152.9831085205078, 26.41014862060547, 177.1981201171875, 84.0302734375, 150.94827270507812, 202.08055114746094, 426.6368408203125, 184.9950408935547, 296.5698547363281, 353.2732849121094, 583.9645385742188, 177.9368438720703, 345.25665283203125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.5457086563110352, 0.02865191176533699, 0.009660398587584496, 0.14272160828113556, 0.05688434839248657, 0.032653454691171646, 0.06331725418567657, 0.043214209377765656, 0.04052463173866272, 0.1960214227437973, 0.15495553612709045, 0.02601071260869503, 0.16830570995807648, 0.0873229056596756, 0.17029429972171783, 0.23504172265529633, 0.4161236584186554, 0.19687841832637787, 0.26037973165512085, 0.33190685510635376, 0.6161143183708191, 0.17115278542041779, 0.3563893437385559]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 658, "cached": true, "scoring": {"normalized": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Moscow is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-25.12500773103995, -0.024666140252647847], "logp": -25.1496738712926, "cache_id": 659, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 575.3573608398438, 29.356595993041992, 10.482144355773926, 150.31640625, 61.959598541259766, 33.22575378417969, 73.96873474121094, 45.55012893676758, 42.7547607421875, 194.24264526367188, 153.03224182128906, 26.38029670715332, 177.20054626464844, 84.02604675292969, 150.96775817871094, 202.09344482421875, 426.6070251464844, 185.06240844726562, 296.5726623535156, 353.2792663574219, 584.0364990234375, 177.91868591308594, 345.28900146484375]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 575.3670043945312, 29.386768341064453, 10.622042655944824, 150.2969970703125, 61.987579345703125, 33.25994873046875, 73.98925018310547, 45.56269836425781, 42.82190704345703, 194.26107788085938, 152.9831085205078, 26.41014862060547, 177.1981201171875, 84.0302734375, 150.94827270507812, 202.08055114746094, 426.6368408203125, 184.9950408935547, 296.5698547363281, 353.2732849121094, 583.9645385742188, 177.9368438720703, 345.25665283203125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.5457086563110352, 0.02865191176533699, 0.009660398587584496, 0.14272160828113556, 0.05688434839248657, 0.032653454691171646, 0.06331725418567657, 0.043214209377765656, 0.04052463173866272, 0.1960214227437973, 0.15495553612709045, 0.02601071260869503, 0.16830570995807648, 0.0873229056596756, 0.17029429972171783, 0.23504172265529633, 0.4161236584186554, 0.19687841832637787, 0.26037973165512085, 0.33190685510635376, 0.6161143183708191, 0.17115278542041779, 0.3563893437385559]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.87500773103995], "logp": -26.87500773103995, "cache_id": 660, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 575.3573608398438, 29.356595993041992, 10.482144355773926, 150.31640625, 61.959598541259766, 33.22575378417969, 73.96873474121094, 45.55012893676758, 42.7547607421875, 194.24264526367188, 153.03224182128906, 26.38029670715332, 177.20054626464844, 84.02604675292969, 150.96775817871094, 202.09344482421875, 426.6070251464844, 185.06240844726562, 296.5726623535156, 353.2792663574219, 584.0364990234375, 177.91868591308594, 345.28900146484375]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 575.3670043945312, 29.386768341064453, 10.622042655944824, 150.2969970703125, 61.987579345703125, 33.25994873046875, 73.98925018310547, 45.56269836425781, 42.82190704345703, 194.26107788085938, 152.9831085205078, 26.41014862060547, 177.1981201171875, 84.0302734375, 150.94827270507812, 202.08055114746094, 426.6368408203125, 184.9950408935547, 296.5698547363281, 353.2732849121094, 583.9645385742188, 177.9368438720703, 345.25665283203125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.5457086563110352, 0.02865191176533699, 0.009660398587584496, 0.14272160828113556, 0.05688434839248657, 0.032653454691171646, 0.06331725418567657, 0.043214209377765656, 0.04052463173866272, 0.1960214227437973, 0.15495553612709045, 0.02601071260869503, 0.16830570995807648, 0.0873229056596756, 0.17029429972171783, 0.23504172265529633, 0.4161236584186554, 0.19687841832637787, 0.26037973165512085, 0.33190685510635376, 0.6161143183708191, 0.17115278542041779, 0.3563893437385559]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "log_odds": -1.7253338597473515, "first_log_odds": -1.75, "later_log_odds": 0.02466614025264846, "first_differing_log_odds": -1.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-11-paraphrase-bridge.npz", "array_sha256": "d51a509e8b4393dc5a68515b159e9e4e8b8ca759880efc0ef1326dc6d2125c94", "seconds": 1.1753907497040927, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-baseline", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 661, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.0640398875802171, -0.006271320005742804], "logp": -0.0703112075859599, "cache_id": 662, "trace": []}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.56403988758022], "logp": -19.56403988758022, "cache_id": 663, "trace": []}, "log_odds": -19.49372867999426, "first_log_odds": -19.5, "later_log_odds": 0.006271320005740932, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-baseline.npz", "array_sha256": "ae76f987bda987a2704cc1cebc4147e49f130587b0e75e026c3a4ba3e265b8b8", "seconds": 0.3519173339009285, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-noop", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.325639724731445, "match_error": -9.325639724731445, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 664, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.0640398875802171, -0.006271320005742804], "logp": -0.0703112075859599, "cache_id": 665, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.325639724731445, "match_error": -9.325639724731445, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.56403988758022], "logp": -19.56403988758022, "cache_id": 666, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.325639724731445, "match_error": -9.325639724731445, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.49372867999426, "first_log_odds": -19.5, "later_log_odds": 0.006271320005740932, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-noop.npz", "array_sha256": "7eaa2e775aabd92ac74051be0b326ef138c2f33b1aaf741f8f81aa66d661ef4b", "seconds": 0.3139062081463635, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-j", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.317368507385254, "actual_norm": 9.325639724731445, "relative_norm": 0.04999558873369539, "target_actual_norm": 9.325639724731445, "match_error": 0.0, "scalar": 0.23633096362031059, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 667, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.08024850808795254, -0.00379397227473266], "logp": -0.0840424803626852, "cache_id": 668, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.317368507385254, "actual_norm": 9.325639724731445, "relative_norm": 0.04999558873369539, "target_actual_norm": 9.325639724731445, "match_error": 0.0, "scalar": 0.23633096362031059, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.58024850808795], "logp": -19.58024850808795, "cache_id": 669, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.317368507385254, "actual_norm": 9.325639724731445, "relative_norm": 0.04999558873369539, "target_actual_norm": 9.325639724731445, "match_error": 0.0, "scalar": 0.23633096362031059, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.496206027725265, "first_log_odds": -19.5, "later_log_odds": 0.0037939722747353244, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-j.npz", "array_sha256": "82ed2259dfb49eebf8b6cf6c83e317aba74bc03a540a2f5441940586a615d528", "seconds": 0.3154808753170073, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-opposite", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.318595886230469, "actual_norm": 9.325758934020996, "relative_norm": 0.049996227825359826, "target_actual_norm": 9.325639724731445, "match_error": 0.00011920928955078125, "scalar": 9.318595975244534, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 670, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.05106851265008248, -0.010807331018088265], "logp": -0.06187584366817074, "cache_id": 671, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.318595886230469, "actual_norm": 9.325758934020996, "relative_norm": 0.049996227825359826, "target_actual_norm": 9.325639724731445, "match_error": 0.00011920928955078125, "scalar": 9.318595975244534, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.926068512650083], "logp": -19.926068512650083, "cache_id": 672, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.318595886230469, "actual_norm": 9.325758934020996, "relative_norm": 0.049996227825359826, "target_actual_norm": 9.325639724731445, "match_error": 0.00011920928955078125, "scalar": 9.318595975244534, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.86419266898191, "first_log_odds": -19.875, "later_log_odds": 0.010807331018089172, "first_differing_log_odds": -19.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-opposite.npz", "array_sha256": "086a681354399cd69d1304a350d0e4b19a72c5424f1aba77e718ae81fc01fda8", "seconds": 0.3169572912156582, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-unrelated", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.316035270690918, "actual_norm": 9.325838088989258, "relative_norm": 0.049996652182225015, "target_actual_norm": 9.325639724731445, "match_error": 0.0001983642578125, "scalar": 9.316034611794748, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 673, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.06410738032218194, -0.006179104512534451], "logp": -0.07028648483471639, "cache_id": 674, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.316035270690918, "actual_norm": 9.325838088989258, "relative_norm": 0.049996652182225015, "target_actual_norm": 9.325639724731445, "match_error": 0.0001983642578125, "scalar": 9.316034611794748, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.68910738032218], "logp": -19.68910738032218, "cache_id": 675, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.316035270690918, "actual_norm": 9.325838088989258, "relative_norm": 0.049996652182225015, "target_actual_norm": 9.325639724731445, "match_error": 0.0001983642578125, "scalar": 9.316034611794748, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.618820895487463, "first_log_odds": -19.625, "later_log_odds": 0.006179104512536782, "first_differing_log_odds": -19.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-unrelated.npz", "array_sha256": "63b8735f1493af34456b715b27381cf62e128d384c96aa8efb6df3c9d5b8237c", "seconds": 0.3203487079590559, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-ordinary", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313546180725098, "actual_norm": 9.325631141662598, "relative_norm": 0.04999554271909555, "target_actual_norm": 9.325639724731445, "match_error": -8.58306884765625e-06, "scalar": 9.313544397329679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 676, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.06332615692513423, -0.006345463213977943], "logp": -0.06967162013911217, "cache_id": 677, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313546180725098, "actual_norm": 9.325631141662598, "relative_norm": 0.04999554271909555, "target_actual_norm": 9.325639724731445, "match_error": -8.58306884765625e-06, "scalar": 9.313544397329679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.938326156925136], "logp": -19.938326156925136, "cache_id": 678, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313546180725098, "actual_norm": 9.325631141662598, "relative_norm": 0.04999554271909555, "target_actual_norm": 9.325639724731445, "match_error": -8.58306884765625e-06, "scalar": 9.313544397329679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.868654536786025, "first_log_odds": -19.875, "later_log_odds": 0.006345463213975222, "first_differing_log_odds": -19.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-ordinary.npz", "array_sha256": "6d795448b5a001c7b51d98953b4d06de9091c22d7fff8232d915bc2f31db0fae", "seconds": 0.31561920791864395, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-random0", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313047409057617, "actual_norm": 9.32553482055664, "relative_norm": 0.049995026333030684, "target_actual_norm": 9.325639724731445, "match_error": -0.0001049041748046875, "scalar": 9.313046354436665, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 679, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.08084211278376992, -0.006917311230943882], "logp": -0.0877594240147138, "cache_id": 680, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313047409057617, "actual_norm": 9.32553482055664, "relative_norm": 0.049995026333030684, "target_actual_norm": 9.325639724731445, "match_error": -0.0001049041748046875, "scalar": 9.313046354436665, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.58084211278377], "logp": -19.58084211278377, "cache_id": 681, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.313047409057617, "actual_norm": 9.32553482055664, "relative_norm": 0.049995026333030684, "target_actual_norm": 9.325639724731445, "match_error": -0.0001049041748046875, "scalar": 9.313046354436665, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.493082688769057, "first_log_odds": -19.5, "later_log_odds": 0.0069173112309428575, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-random0.npz", "array_sha256": "3b39b40e3c3ede47703f579686450e2ae190f91a2fbf9d26a9fec7c181f373a8", "seconds": 0.3170323329977691, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-random1", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.309313774108887, "actual_norm": 9.325749397277832, "relative_norm": 0.049996176698026674, "target_actual_norm": 9.325639724731445, "match_error": 0.00010967254638671875, "scalar": 9.309313256144833, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 682, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.08083388505649555, -0.006291199786189042], "logp": -0.0871250848426846, "cache_id": 683, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.309313774108887, "actual_norm": 9.325749397277832, "relative_norm": 0.049996176698026674, "target_actual_norm": 9.325639724731445, "match_error": 0.00010967254638671875, "scalar": 9.309313256144833, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.580833885056496], "logp": -19.580833885056496, "cache_id": 684, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.309313774108887, "actual_norm": 9.325749397277832, "relative_norm": 0.049996176698026674, "target_actual_norm": 9.325639724731445, "match_error": 0.00010967254638671875, "scalar": 9.309313256144833, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.49370880021381, "first_log_odds": -19.5, "later_log_odds": 0.006291199786190305, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-random1.npz", "array_sha256": "ef16c6e1ba5ed1e7100efc91a86275663fe000f61806a867b7d65f0a3745b74b", "seconds": 0.31453045876696706, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-random2", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "RUB<|im_end|>", "output_text": "RUB", "output_ids": [49, 4493, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.32492733001709, "actual_norm": 9.325671195983887, "relative_norm": 0.049995757453894805, "target_actual_norm": 9.325639724731445, "match_error": 3.147125244140625e-05, "scalar": 9.324928234884283, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 685, "cached": true, "scoring": {"normalized": "RUB", "bare_format": true, "malformed": false, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-0.10217944959591556, -0.0050175869443762405], "logp": -0.10719703654029179, "cache_id": 686, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.32492733001709, "actual_norm": 9.325671195983887, "relative_norm": 0.049995757453894805, "target_actual_norm": 9.325639724731445, "match_error": 3.147125244140625e-05, "scalar": 9.324928234884283, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-19.602179449595916], "logp": -19.602179449595916, "cache_id": 687, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.5292510986328, "strength_one_norm": 39.42509078979492, "target_shrink_scale": 0.23633096362031059, "requested_norm": 9.32492733001709, "actual_norm": 9.325671195983887, "relative_norm": 0.049995757453894805, "target_actual_norm": 9.325639724731445, "match_error": 3.147125244140625e-05, "scalar": 9.324928234884283, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -19.494982413055624, "first_log_odds": -19.5, "later_log_odds": 0.005017586944376262, "first_differing_log_odds": -19.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-random2.npz", "array_sha256": "619031a1ec5e96fde94cdad43a3de07396d3233758413a006c31b7791836bf2d", "seconds": 0.3137196246534586, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-main-bridge", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 73.64644622802734, 513.155029296875, 98.9384765625, 133.67266845703125, 1.6997785568237305, 10.440433502197266, 9.071504592895508, 55.22721481323242, 61.01800537109375, 9.297075271606445, 182.91476440429688, 160.63499450683594, 46.46152877807617, 223.74252319335938, 1.7796568870544434, 41.01746368408203, 50.494075775146484, 38.95417785644531, 159.42222595214844, 166.1367950439453, 404.35418701171875, 144.8641815185547, 345.2865295410156, 350.3744812011719, 541.5799560546875, 181.47796630859375, 681.5394897460938]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 73.68767547607422, 513.1405639648438, 98.90680694580078, 133.67442321777344, 1.7268309593200684, 10.552633285522461, 9.164358139038086, 55.26884841918945, 61.0654182434082, 9.409293174743652, 182.94566345214844, 160.64012145996094, 46.52865982055664, 223.7521514892578, 1.8212753534317017, 41.08339309692383, 50.50746536254883, 38.96992492675781, 159.43589782714844, 166.11753845214844, 404.3368225097656, 144.86439514160156, 345.2956848144531, 350.3338928222656, 541.5906982421875, 181.42311096191406, 681.5987548828125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.0653449222445488, 0.5262551307678223, 0.10297024995088577, 0.12666231393814087, 0.0016063558869063854, 0.010241656564176083, 0.009120156057178974, 0.04878556355834007, 0.05980023369193077, 0.009015345014631748, 0.18618015944957733, 0.16708111763000488, 0.04709366336464882, 0.21504056453704834, 0.0017319591715931892, 0.0330626480281353, 0.050630658864974976, 0.04364178702235222, 0.17585697770118713, 0.19753266870975494, 0.39939555525779724, 0.1525595635175705, 0.3035046458244324, 0.3396500051021576, 0.5802513957023621, 0.17524667084217072, 0.6557306051254272]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}], "cache_id": 688, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-13.244828462146387, -0.00340532438592086], "logp": -13.248233786532309, "cache_id": 689, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 73.64644622802734, 513.155029296875, 98.9384765625, 133.67266845703125, 1.6997785568237305, 10.440433502197266, 9.071504592895508, 55.22721481323242, 61.01800537109375, 9.297075271606445, 182.91476440429688, 160.63499450683594, 46.46152877807617, 223.74252319335938, 1.7796568870544434, 41.01746368408203, 50.494075775146484, 38.95417785644531, 159.42222595214844, 166.1367950439453, 404.35418701171875, 144.8641815185547, 345.2865295410156, 350.3744812011719, 541.5799560546875, 181.47796630859375, 681.5394897460938]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 73.68767547607422, 513.1405639648438, 98.90680694580078, 133.67442321777344, 1.7268309593200684, 10.552633285522461, 9.164358139038086, 55.26884841918945, 61.0654182434082, 9.409293174743652, 182.94566345214844, 160.64012145996094, 46.52865982055664, 223.7521514892578, 1.8212753534317017, 41.08339309692383, 50.50746536254883, 38.96992492675781, 159.43589782714844, 166.11753845214844, 404.3368225097656, 144.86439514160156, 345.2956848144531, 350.3338928222656, 541.5906982421875, 181.42311096191406, 681.5987548828125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.0653449222445488, 0.5262551307678223, 0.10297024995088577, 0.12666231393814087, 0.0016063558869063854, 0.010241656564176083, 0.009120156057178974, 0.04878556355834007, 0.05980023369193077, 0.009015345014631748, 0.18618015944957733, 0.16708111763000488, 0.04709366336464882, 0.21504056453704834, 0.0017319591715931892, 0.0330626480281353, 0.050630658864974976, 0.04364178702235222, 0.17585697770118713, 0.19753266870975494, 0.39939555525779724, 0.1525595635175705, 0.3035046458244324, 0.3396500051021576, 0.5802513957023621, 0.17524667084217072, 0.6557306051254272]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-5.869828462146386], "logp": -5.869828462146386, "cache_id": 690, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 176.31739807128906, 201.15457153320312, 265.70245361328125, 216.95217895507812, 90.12483215332031, 73.64644622802734, 513.155029296875, 98.9384765625, 133.67266845703125, 1.6997785568237305, 10.440433502197266, 9.071504592895508, 55.22721481323242, 61.01800537109375, 9.297075271606445, 182.91476440429688, 160.63499450683594, 46.46152877807617, 223.74252319335938, 1.7796568870544434, 41.01746368408203, 50.494075775146484, 38.95417785644531, 159.42222595214844, 166.1367950439453, 404.35418701171875, 144.8641815185547, 345.2865295410156, 350.3744812011719, 541.5799560546875, 181.47796630859375, 681.5394897460938]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 176.34971618652344, 201.0691680908203, 265.71124267578125, 216.94459533691406, 90.12816619873047, 73.68767547607422, 513.1405639648438, 98.90680694580078, 133.67442321777344, 1.7268309593200684, 10.552633285522461, 9.164358139038086, 55.26884841918945, 61.0654182434082, 9.409293174743652, 182.94566345214844, 160.64012145996094, 46.52865982055664, 223.7521514892578, 1.8212753534317017, 41.08339309692383, 50.50746536254883, 38.96992492675781, 159.43589782714844, 166.11753845214844, 404.3368225097656, 144.86439514160156, 345.2956848144531, 350.3338928222656, 541.5906982421875, 181.42311096191406, 681.5987548828125]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.16188284754753113, 0.1764143407344818, 0.21617019176483154, 0.18568727374076843, 0.07942100614309311, 0.0653449222445488, 0.5262551307678223, 0.10297024995088577, 0.12666231393814087, 0.0016063558869063854, 0.010241656564176083, 0.009120156057178974, 0.04878556355834007, 0.05980023369193077, 0.009015345014631748, 0.18618015944957733, 0.16708111763000488, 0.04709366336464882, 0.21504056453704834, 0.0017319591715931892, 0.0330626480281353, 0.050630658864974976, 0.04364178702235222, 0.17585697770118713, 0.19753266870975494, 0.39939555525779724, 0.1525595635175705, 0.3035046458244324, 0.3396500051021576, 0.5802513957023621, 0.17524667084217072, 0.6557306051254272]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "log_odds": 7.3784053243859224, "first_log_odds": 7.375000000000001, "later_log_odds": 0.00340532438592156, "first_differing_log_odds": 7.375000000000001, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-main-bridge.npz", "array_sha256": "66954ca913e60aa70617e80d7c55d3c893c1dd490ed14065bffcb671159213b4", "seconds": 0.28776508290320635, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-baseline", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 691, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.125042477567632, -0.002523515394633865], "logp": -16.127565992962268, "cache_id": 692, "trace": []}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.625042477567632], "logp": -28.625042477567632, "cache_id": 693, "trace": []}, "log_odds": -12.497476484605365, "first_log_odds": -12.5, "later_log_odds": 0.002523515394635467, "first_differing_log_odds": -12.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-baseline.npz", "array_sha256": "877996b455c592c9ceb9df99cc29eeac2a95a122dbf50f950199a92a88bd637a", "seconds": 1.2090152497403324, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-noop", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.343810081481934, "match_error": -9.343810081481934, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 694, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.125042477567632, -0.002523515394633865], "logp": -16.127565992962268, "cache_id": 695, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.343810081481934, "match_error": -9.343810081481934, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.625042477567632], "logp": -28.625042477567632, "cache_id": 696, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.343810081481934, "match_error": -9.343810081481934, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.497476484605365, "first_log_odds": -12.5, "later_log_odds": 0.002523515394635467, "first_differing_log_odds": -12.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-noop.npz", "array_sha256": "5a9a226c123572d6197c275e8c17ab92fd323c3f752c8d4d6a63e3be23b30ab2", "seconds": 1.1965229166671634, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-j", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.348015785217285, "actual_norm": 9.343810081481934, "relative_norm": 0.049977514009442385, "target_actual_norm": 9.343810081481934, "match_error": 0.0, "scalar": 0.225366210488445, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 697, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.62504525781952, -0.0024422982678775156], "logp": -16.627487556087395, "cache_id": 698, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.348015785217285, "actual_norm": 9.343810081481934, "relative_norm": 0.049977514009442385, "target_actual_norm": 9.343810081481934, "match_error": 0.0, "scalar": 0.225366210488445, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.25004525781952], "logp": -28.25004525781952, "cache_id": 699, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.348015785217285, "actual_norm": 9.343810081481934, "relative_norm": 0.049977514009442385, "target_actual_norm": 9.343810081481934, "match_error": 0.0, "scalar": 0.225366210488445, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -11.622557701732124, "first_log_odds": -11.625, "later_log_odds": 0.0024422982678764527, "first_differing_log_odds": -11.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-j.npz", "array_sha256": "9cbec833a537ef0ae3ef5530ceee8efa572cf2db2655c399d99a93abf830c142", "seconds": 1.1930669997818768, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-opposite", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.344808578491211, "actual_norm": 9.34382152557373, "relative_norm": 0.04997757522079547, "target_actual_norm": 9.343810081481934, "match_error": 1.1444091796875e-05, "scalar": 9.344808108071447, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 700, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-15.500048435046596, -0.003297577261204904], "logp": -15.5033460123078, "cache_id": 701, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.344808578491211, "actual_norm": 9.34382152557373, "relative_norm": 0.04997757522079547, "target_actual_norm": 9.343810081481934, "match_error": 1.1444091796875e-05, "scalar": 9.344808108071447, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.812548435046594], "logp": -28.812548435046594, "cache_id": 702, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.344808578491211, "actual_norm": 9.34382152557373, "relative_norm": 0.04997757522079547, "target_actual_norm": 9.343810081481934, "match_error": 1.1444091796875e-05, "scalar": 9.344808108071447, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -13.309202422738794, "first_log_odds": -13.312499999999998, "later_log_odds": 0.0032975772612040544, "first_differing_log_odds": -13.312499999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-opposite.npz", "array_sha256": "4a377d63155191a9061832e5e17a84e61130e3c18a160ab57acea732ba4e43f3", "seconds": 1.1960010826587677, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-unrelated", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.340885162353516, "actual_norm": 9.342915534973145, "relative_norm": 0.04997272932200918, "target_actual_norm": 9.343810081481934, "match_error": -0.0008945465087890625, "scalar": 9.340887289326929, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 703, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-15.875048121233734, -0.0021631706381689633], "logp": -15.877211291871903, "cache_id": 704, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.340885162353516, "actual_norm": 9.342915534973145, "relative_norm": 0.04997272932200918, "target_actual_norm": 9.343810081481934, "match_error": -0.0008945465087890625, "scalar": 9.340887289326929, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.500048121233732], "logp": -28.500048121233732, "cache_id": 705, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.340885162353516, "actual_norm": 9.342915534973145, "relative_norm": 0.04997272932200918, "target_actual_norm": 9.343810081481934, "match_error": -0.0008945465087890625, "scalar": 9.340887289326929, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.622836829361828, "first_log_odds": -12.624999999999998, "later_log_odds": 0.0021631706381697313, "first_differing_log_odds": -12.624999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-unrelated.npz", "array_sha256": "96e64ffd9509cd0e1197622379e9f8c4273bb8d5f37db51013a59cab362a7383", "seconds": 1.2030902081169188, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-ordinary", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.334754943847656, "actual_norm": 9.343443870544434, "relative_norm": 0.0499755552461435, "target_actual_norm": 9.343810081481934, "match_error": -0.0003662109375, "scalar": 9.334756554562773, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 706, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.00005283602393, -0.002028403446041246], "logp": -16.00208123946997, "cache_id": 707, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.334754943847656, "actual_norm": 9.343443870544434, "relative_norm": 0.0499755552461435, "target_actual_norm": 9.343810081481934, "match_error": -0.0003662109375, "scalar": 9.334756554562773, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.50005283602393], "logp": -28.50005283602393, "cache_id": 708, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.334754943847656, "actual_norm": 9.343443870544434, "relative_norm": 0.0499755552461435, "target_actual_norm": 9.343810081481934, "match_error": -0.0003662109375, "scalar": 9.334756554562773, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.49797159655396, "first_log_odds": -12.5, "later_log_odds": 0.0020284034460402722, "first_differing_log_odds": -12.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-ordinary.npz", "array_sha256": "88a6e2506bd84d05769994928ce1068acd218b3de1b9e88ad87a4cb39cdbf06e", "seconds": 1.1979391667991877, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-random0", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.339818954467773, "actual_norm": 9.343789100646973, "relative_norm": 0.04997740178862838, "target_actual_norm": 9.343810081481934, "match_error": -2.09808349609375e-05, "scalar": 9.339817975123879, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 709, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-15.875059786013933, -0.0024607983524770486], "logp": -15.87752058436641, "cache_id": 710, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.339818954467773, "actual_norm": 9.343789100646973, "relative_norm": 0.04997740178862838, "target_actual_norm": 9.343810081481934, "match_error": -2.09808349609375e-05, "scalar": 9.339817975123879, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.375059786013935], "logp": -28.375059786013935, "cache_id": 711, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.339818954467773, "actual_norm": 9.343789100646973, "relative_norm": 0.04997740178862838, "target_actual_norm": 9.343810081481934, "match_error": -2.09808349609375e-05, "scalar": 9.339817975123879, "random_seed": 20272108, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.497539201647525, "first_log_odds": -12.500000000000002, "later_log_odds": 0.0024607983524767008, "first_differing_log_odds": -12.500000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-random0.npz", "array_sha256": "0cdd1d83f0c5c42d0b49b2b4da5863cc1b920685e992a229d1943d3669f129ae", "seconds": 1.1292213327251375, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-random1", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.332462310791016, "actual_norm": 9.343782424926758, "relative_norm": 0.049977366082005745, "target_actual_norm": 9.343810081481934, "match_error": -2.765655517578125e-05, "scalar": 9.33246198450206, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 712, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-15.875046809184411, -0.002758916570951511], "logp": -15.877805725755362, "cache_id": 713, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.332462310791016, "actual_norm": 9.343782424926758, "relative_norm": 0.049977366082005745, "target_actual_norm": 9.343810081481934, "match_error": -2.765655517578125e-05, "scalar": 9.33246198450206, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.43754680918441], "logp": -28.43754680918441, "cache_id": 714, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.332462310791016, "actual_norm": 9.343782424926758, "relative_norm": 0.049977366082005745, "target_actual_norm": 9.343810081481934, "match_error": -2.765655517578125e-05, "scalar": 9.33246198450206, "random_seed": 20272109, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.55974108342905, "first_log_odds": -12.5625, "later_log_odds": 0.0027589165709507313, "first_differing_log_odds": -12.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-random1.npz", "array_sha256": "4ca3649869b4c6669d04725d802b5b4ce96edbb44e428c753af3b2cfe7afc6c6", "seconds": 1.2070907917805016, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-random2", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.342031478881836, "actual_norm": 9.343822479248047, "relative_norm": 0.049977580321741565, "target_actual_norm": 9.343810081481934, "match_error": 1.239776611328125e-05, "scalar": 9.342032346619362, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 715, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-16.250041186071496, -0.0025375103143147506], "logp": -16.25257869638581, "cache_id": 716, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.342031478881836, "actual_norm": 9.343822479248047, "relative_norm": 0.049977580321741565, "target_actual_norm": 9.343810081481934, "match_error": 1.239776611328125e-05, "scalar": 9.342032346619362, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-28.562541186071496], "logp": -28.562541186071496, "cache_id": 717, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 186.9602813720703, "strength_one_norm": 41.479217529296875, "target_shrink_scale": 0.225366210488445, "requested_norm": 9.342031478881836, "actual_norm": 9.343822479248047, "relative_norm": 0.049977580321741565, "target_actual_norm": 9.343810081481934, "match_error": 1.239776611328125e-05, "scalar": 9.342032346619362, "random_seed": 20272110, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "log_odds": -12.309962489685685, "first_log_odds": -12.3125, "later_log_odds": 0.002537510314315483, "first_differing_log_odds": -12.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-random2.npz", "array_sha256": "be2e0352cd7521ca235f8168de8509205a989e4db8ae71380b4fa8da55f6cd01", "seconds": 1.1940174167975783, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-12-paraphrase-bridge", "case_id": "eval-12", "city": "Saint Petersburg", "country": "Russia", "code": "RUB", "target_country": "Turkey", "target_code": "TRY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.<|im_end|>", "output_text": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "output_ids": [785, 11413, 1483, 304, 14205, 53948, 374, 279, 8522, 10273, 273, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 49, 4493, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 86.07447814941406, 521.572998046875, 5.419095516204834, 10.913360595703125, 119.88877868652344, 24.1624698638916, 1.7649906873703003, 73.40362548828125, 55.67398452758789, 30.09644317626953, 191.162841796875, 147.57244873046875, 26.11960792541504, 162.89187622070312, 101.65998077392578, 157.64891052246094, 201.13331604003906, 424.95257568359375, 188.88888549804688, 309.2301025390625, 349.0317687988281, 596.5584106445312, 183.29351806640625, 396.5412902832031]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 86.13054656982422, 521.5851440429688, 5.545742034912109, 11.07938003540039, 119.97085571289062, 24.21866226196289, 1.7568804025650024, 73.37773132324219, 55.67371368408203, 30.121809005737305, 191.1409454345703, 147.5206298828125, 26.186614990234375, 162.92190551757812, 101.67391967773438, 157.71514892578125, 201.1715850830078, 424.9479675292969, 188.92596435546875, 309.20880126953125, 349.0478515625, 596.5802001953125, 183.2911834716797, 396.5841979980469]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.07709363102912903, 0.534460186958313, 0.005537228658795357, 0.010062535293400288, 0.11080353707075119, 0.02208871953189373, 0.0017647107597440481, 0.0633556991815567, 0.05289814993739128, 0.028519030660390854, 0.19297178089618683, 0.1513446569442749, 0.025624318048357964, 0.15304066240787506, 0.10633412003517151, 0.17795787751674652, 0.23397167026996613, 0.415481835603714, 0.19926269352436066, 0.26956576108932495, 0.33021098375320435, 0.6306557059288025, 0.17654234170913696, 0.4167795777320862]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 718, "cached": true, "scoring": {"normalized": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **RUB**.", "bare_format": false, "malformed": true, "first_code": "RUB", "all_codes": ["RUB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Saint Petersburg is the Russian ruble, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Russian", "start": 45, "end": 52}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [49, 4493], "tokens": ["R", "UB"], "log_probs": [-28.12500610160795, -0.0019427890126535907], "logp": -28.126948890620604, "cache_id": 719, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 86.07447814941406, 521.572998046875, 5.419095516204834, 10.913360595703125, 119.88877868652344, 24.1624698638916, 1.7649906873703003, 73.40362548828125, 55.67398452758789, 30.09644317626953, 191.162841796875, 147.57244873046875, 26.11960792541504, 162.89187622070312, 101.65998077392578, 157.64891052246094, 201.13331604003906, 424.95257568359375, 188.88888549804688, 309.2301025390625, 349.0317687988281, 596.5584106445312, 183.29351806640625, 396.5412902832031]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 86.13054656982422, 521.5851440429688, 5.545742034912109, 11.07938003540039, 119.97085571289062, 24.21866226196289, 1.7568804025650024, 73.37773132324219, 55.67371368408203, 30.121809005737305, 191.1409454345703, 147.5206298828125, 26.186614990234375, 162.92190551757812, 101.67391967773438, 157.71514892578125, 201.1715850830078, 424.9479675292969, 188.92596435546875, 309.20880126953125, 349.0478515625, 596.5802001953125, 183.2911834716797, 396.5841979980469]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.07709363102912903, 0.534460186958313, 0.005537228658795357, 0.010062535293400288, 0.11080353707075119, 0.02208871953189373, 0.0017647107597440481, 0.0633556991815567, 0.05289814993739128, 0.028519030660390854, 0.19297178089618683, 0.1513446569442749, 0.025624318048357964, 0.15304066240787506, 0.10633412003517151, 0.17795787751674652, 0.23397167026996613, 0.415481835603714, 0.19926269352436066, 0.26956576108932495, 0.33021098375320435, 0.6306557059288025, 0.17654234170913696, 0.4167795777320862]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-25.75000610160795], "logp": -25.75000610160795, "cache_id": 720, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[98.62864685058594, 195.30047607421875, 180.06979370117188, 127.76617431640625, 102.4910888671875, 169.23922729492188, 176.3018798828125, 177.21498107910156, 86.07447814941406, 521.572998046875, 5.419095516204834, 10.913360595703125, 119.88877868652344, 24.1624698638916, 1.7649906873703003, 73.40362548828125, 55.67398452758789, 30.09644317626953, 191.162841796875, 147.57244873046875, 26.11960792541504, 162.89187622070312, 101.65998077392578, 157.64891052246094, 201.13331604003906, 424.95257568359375, 188.88888549804688, 309.2301025390625, 349.0317687988281, 596.5584106445312, 183.29351806640625, 396.5412902832031]], "actual_norm": [[98.6502685546875, 195.28424072265625, 180.05245971679688, 127.74275207519531, 102.50137329101562, 169.21096801757812, 176.35838317871094, 177.22642517089844, 86.13054656982422, 521.5851440429688, 5.545742034912109, 11.07938003540039, 119.97085571289062, 24.21866226196289, 1.7568804025650024, 73.37773132324219, 55.67371368408203, 30.121809005737305, 191.1409454345703, 147.5206298828125, 26.186614990234375, 162.92190551757812, 101.67391967773438, 157.71514892578125, 201.1715850830078, 424.9479675292969, 188.92596435546875, 309.20880126953125, 349.0478515625, 596.5802001953125, 183.2911834716797, 396.5841979980469]], "relative_norm": [[0.0057577104307711124, 0.20715337991714478, 0.19919414818286896, 0.11137832701206207, 0.09330446273088455, 0.14699964225292206, 0.1568511575460434, 0.15743133425712585, 0.07709363102912903, 0.534460186958313, 0.005537228658795357, 0.010062535293400288, 0.11080353707075119, 0.02208871953189373, 0.0017647107597440481, 0.0633556991815567, 0.05289814993739128, 0.028519030660390854, 0.19297178089618683, 0.1513446569442749, 0.025624318048357964, 0.15304066240787506, 0.10633412003517151, 0.17795787751674652, 0.23397167026996613, 0.415481835603714, 0.19926269352436066, 0.26956576108932495, 0.33021098375320435, 0.6306557059288025, 0.17654234170913696, 0.4167795777320862]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "log_odds": 2.376942789012652, "first_log_odds": 2.375, "later_log_odds": 0.001942789012652213, "first_differing_log_odds": 2.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-12-paraphrase-bridge.npz", "array_sha256": "2ec818a21da674b8e5cc82fdf40cbfc0151071890b98a9dc9700becc2c99b5a7", "seconds": 1.2263271249830723, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-baseline", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 721, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.293178611326512], "logp": -4.293178611326512, "cache_id": 722, "trace": []}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.043178611326512, -2.84371135507584], "logp": -17.886889966402354, "cache_id": 723, "trace": []}, "log_odds": -13.593711355075841, "first_log_odds": -10.75, "later_log_odds": -2.8437113550758415, "first_differing_log_odds": -10.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-baseline.npz", "array_sha256": "4cdc1f72ae73f7d633b70f47812eebe11da36646f2394fc7d6fa58c96526bb5e", "seconds": 0.31874804198741913, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3456977408, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-noop", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.228175163269043, "match_error": -9.228175163269043, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 724, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.293178611326512], "logp": -4.293178611326512, "cache_id": 725, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.228175163269043, "match_error": -9.228175163269043, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.043178611326512, -2.84371135507584], "logp": -17.886889966402354, "cache_id": 726, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.228175163269043, "match_error": -9.228175163269043, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.593711355075841, "first_log_odds": -10.75, "later_log_odds": -2.8437113550758415, "first_differing_log_odds": -10.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-noop.npz", "array_sha256": "b66ad0708468f4d13e3978dcf9ba7949d76d84c3fe82f8bdda8054541ee8364d", "seconds": 0.2947335420176387, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457010176, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-j", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230999946594238, "actual_norm": 9.228175163269043, "relative_norm": 0.04998469637433108, "target_actual_norm": 9.228175163269043, "match_error": 0.0, "scalar": 0.23252488817700726, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 727, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.283620483629114], "logp": -4.283620483629114, "cache_id": 728, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230999946594238, "actual_norm": 9.228175163269043, "relative_norm": 0.04998469637433108, "target_actual_norm": 9.228175163269043, "match_error": 0.0, "scalar": 0.23252488817700726, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-14.658620483629113, -2.5010720409790625], "logp": -17.159692524608175, "cache_id": 729, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230999946594238, "actual_norm": 9.228175163269043, "relative_norm": 0.04998469637433108, "target_actual_norm": 9.228175163269043, "match_error": 0.0, "scalar": 0.23252488817700726, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.87607204097906, "first_log_odds": -10.375, "later_log_odds": -2.5010720409790608, "first_differing_log_odds": -10.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-j.npz", "array_sha256": "ba0e5ddb00f450aca765daa6b9edf4278ce11809e622ed7155caf5d877f9c297", "seconds": 0.2907866248860955, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457010176, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-opposite", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.227330207824707, "actual_norm": 9.228626251220703, "relative_norm": 0.04998713970618196, "target_actual_norm": 9.228175163269043, "match_error": 0.00045108795166015625, "scalar": 9.227330298599554, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 730, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.545479440084978], "logp": -4.545479440084978, "cache_id": 731, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.227330207824707, "actual_norm": 9.228626251220703, "relative_norm": 0.04998713970618196, "target_actual_norm": 9.228175163269043, "match_error": 0.00045108795166015625, "scalar": 9.227330298599554, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.295479440084977, -2.727097484753377], "logp": -18.022576924838354, "cache_id": 732, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.227330207824707, "actual_norm": 9.228626251220703, "relative_norm": 0.04998713970618196, "target_actual_norm": 9.228175163269043, "match_error": 0.00045108795166015625, "scalar": 9.227330298599554, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.477097484753376, "first_log_odds": -10.75, "later_log_odds": -2.7270974847533758, "first_differing_log_odds": -10.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-opposite.npz", "array_sha256": "757e1551833b5a292eb6bd3936e0f27bcdaf2d7aec72e61d0d16fc89448c6e16", "seconds": 0.2898029168136418, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457010176, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-unrelated", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23275375366211, "actual_norm": 9.227494239807129, "relative_norm": 0.04998100813132575, "target_actual_norm": 9.228175163269043, "match_error": -0.0006809234619140625, "scalar": 9.232751513562107, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 733, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-3.9346086593000784], "logp": -3.9346086593000784, "cache_id": 734, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23275375366211, "actual_norm": 9.227494239807129, "relative_norm": 0.04998100813132575, "target_actual_norm": 9.228175163269043, "match_error": -0.0006809234619140625, "scalar": 9.232751513562107, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-14.684608659300078, -2.7285162078755376], "logp": -17.413124867175615, "cache_id": 735, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23275375366211, "actual_norm": 9.227494239807129, "relative_norm": 0.04998100813132575, "target_actual_norm": 9.228175163269043, "match_error": -0.0006809234619140625, "scalar": 9.232751513562107, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.478516207875536, "first_log_odds": -10.75, "later_log_odds": -2.7285162078755363, "first_differing_log_odds": -10.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-unrelated.npz", "array_sha256": "b51d5bea33ce5f0bcb47a5e2e52c808eb49f8c51f806aa92fd09a7441a073680", "seconds": 0.2911869171075523, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457010176, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-ordinary", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23254108428955, "actual_norm": 9.22803020477295, "relative_norm": 0.04998391120215068, "target_actual_norm": 9.228175163269043, "match_error": -0.00014495849609375, "scalar": 9.232540297394735, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 736, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.5312641292385525], "logp": -4.5312641292385525, "cache_id": 737, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23254108428955, "actual_norm": 9.22803020477295, "relative_norm": 0.04998391120215068, "target_actual_norm": 9.228175163269043, "match_error": -0.00014495849609375, "scalar": 9.232540297394735, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.031264129238552, -2.7281067047992376], "logp": -17.75937083403779, "cache_id": 738, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.23254108428955, "actual_norm": 9.22803020477295, "relative_norm": 0.04998391120215068, "target_actual_norm": 9.228175163269043, "match_error": -0.00014495849609375, "scalar": 9.232540297394735, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.228106704799236, "first_log_odds": -10.5, "later_log_odds": -2.7281067047992362, "first_differing_log_odds": -10.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-ordinary.npz", "array_sha256": "a9b1262bbdd4ba7dcdcda1f81ed63ac8b18e6ebc0d779828e6f5b10552b5a6ec", "seconds": 0.2901425417512655, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457042944, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-random0", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.229214668273926, "actual_norm": 9.228109359741211, "relative_norm": 0.04998433994748603, "target_actual_norm": 9.228175163269043, "match_error": -6.580352783203125e-05, "scalar": 9.229213642758623, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 739, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.167413617553189], "logp": -4.167413617553189, "cache_id": 740, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.229214668273926, "actual_norm": 9.228109359741211, "relative_norm": 0.04998433994748603, "target_actual_norm": 9.228175163269043, "match_error": -6.580352783203125e-05, "scalar": 9.229213642758623, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-14.917413617553189, -2.7288530974110365], "logp": -17.646266714964227, "cache_id": 741, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.229214668273926, "actual_norm": 9.228109359741211, "relative_norm": 0.04998433994748603, "target_actual_norm": 9.228175163269043, "match_error": -6.580352783203125e-05, "scalar": 9.229213642758623, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.478853097411038, "first_log_odds": -10.75, "later_log_odds": -2.7288530974110383, "first_differing_log_odds": -10.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-random0.npz", "array_sha256": "e3e4c4254d485445c9f6c199f5871dc46e7a10e47041e6edc9508471ae005476", "seconds": 0.2921018749475479, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457042944, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-random1", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230005264282227, "actual_norm": 9.228198051452637, "relative_norm": 0.049984820348885885, "target_actual_norm": 9.228175163269043, "match_error": 2.288818359375e-05, "scalar": 9.230005703386269, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 742, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.782364371668169], "logp": -4.782364371668169, "cache_id": 743, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230005264282227, "actual_norm": 9.228198051452637, "relative_norm": 0.049984820348885885, "target_actual_norm": 9.228175163269043, "match_error": 2.288818359375e-05, "scalar": 9.230005703386269, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.40736437166817, -2.6148532265329227], "logp": -18.02221759820109, "cache_id": 744, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.230005264282227, "actual_norm": 9.228198051452637, "relative_norm": 0.049984820348885885, "target_actual_norm": 9.228175163269043, "match_error": 2.288818359375e-05, "scalar": 9.230005703386269, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.239853226532922, "first_log_odds": -10.625, "later_log_odds": -2.6148532265329223, "first_differing_log_odds": -10.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-random1.npz", "array_sha256": "5bfb892db81b13ccf0f30c188003e7c5a4c876d78e9d15b3ec8d1867556d7bb9", "seconds": 0.29027854232117534, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457042944, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-random2", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.214262008666992, "actual_norm": 9.228239059448242, "relative_norm": 0.049985042469963235, "target_actual_norm": 9.228175163269043, "match_error": 6.389617919921875e-05, "scalar": 9.2142612982434, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 745, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-4.5362544095093815], "logp": -4.5362544095093815, "cache_id": 746, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.214262008666992, "actual_norm": 9.228239059448242, "relative_norm": 0.049985042469963235, "target_actual_norm": 9.228175163269043, "match_error": 6.389617919921875e-05, "scalar": 9.2142612982434, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-15.161254409509382, -2.958271827623342], "logp": -18.119526237132725, "cache_id": 747, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 184.62001037597656, "strength_one_norm": 39.698978424072266, "target_shrink_scale": 0.23252488817700726, "requested_norm": 9.214262008666992, "actual_norm": 9.228239059448242, "relative_norm": 0.049985042469963235, "target_actual_norm": 9.228175163269043, "match_error": 6.389617919921875e-05, "scalar": 9.2142612982434, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.583271827623342, "first_log_odds": -10.625, "later_log_odds": -2.958271827623342, "first_differing_log_odds": -10.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-random2.npz", "array_sha256": "42e151d4df2872f997e8579ff16254921acfdf8044cb8934cf545538d1dccf13", "seconds": 0.2912619998678565, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457042944, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-main-bridge", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THY<|im_end|>", "output_text": "THY", "output_ids": [3617, 56, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 998.4506225585938, 78.0566635131836, 197.07781982421875, 67.62660217285156, 7.613514423370361, 17.8790340423584, 92.05768585205078, 15.205554962158203, 7.409041881561279, 97.7878189086914, 54.19824981689453, 25.320829391479492, 25.35968780517578, 75.88646697998047, 163.03302001953125, 123.41985321044922, 164.9969482421875, 146.29742431640625, 75.9881362915039, 278.145263671875, 112.75767517089844, 117.22711181640625, 219.6559600830078, 202.41775512695312, 66.92359924316406, 1170.4088134765625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 998.400634765625, 78.01586151123047, 197.13389587402344, 67.68624114990234, 7.727022647857666, 17.911195755004883, 92.10831451416016, 15.27455997467041, 7.607632637023926, 97.83941650390625, 54.20945358276367, 25.386960983276367, 25.40325355529785, 75.90375518798828, 163.0804443359375, 123.3778076171875, 165.00119018554688, 146.33189392089844, 76.0130615234375, 278.2159423828125, 112.83272552490234, 117.33250427246094, 219.61801147460938, 202.34776306152344, 66.94676208496094, 1170.4490966796875]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.9259202480316162, 0.08017925918102264, 0.19209080934524536, 0.06361857056617737, 0.007567114662379026, 0.017343170940876007, 0.0807785913348198, 0.014796692878007889, 0.007238495629280806, 0.10004221647977829, 0.05593104660511017, 0.025312207639217377, 0.02431764453649521, 0.07245571911334991, 0.1315336972475052, 0.12484225630760193, 0.18530867993831635, 0.16127800941467285, 0.08993591368198395, 0.2760690748691559, 0.1197303980588913, 0.10385147482156754, 0.21126008033752441, 0.21669329702854156, 0.06467969715595245, 1.0695146322250366]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 748, "cached": true, "scoring": {"normalized": "THY", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-22.93214930379883], "logp": -22.93214930379883, "cache_id": 749, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 998.4506225585938, 78.0566635131836, 197.07781982421875, 67.62660217285156, 7.613514423370361, 17.8790340423584, 92.05768585205078, 15.205554962158203, 7.409041881561279, 97.7878189086914, 54.19824981689453, 25.320829391479492, 25.35968780517578, 75.88646697998047, 163.03302001953125, 123.41985321044922, 164.9969482421875, 146.29742431640625, 75.9881362915039, 278.145263671875, 112.75767517089844, 117.22711181640625, 219.6559600830078, 202.41775512695312, 66.92359924316406, 1170.4088134765625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 998.400634765625, 78.01586151123047, 197.13389587402344, 67.68624114990234, 7.727022647857666, 17.911195755004883, 92.10831451416016, 15.27455997467041, 7.607632637023926, 97.83941650390625, 54.20945358276367, 25.386960983276367, 25.40325355529785, 75.90375518798828, 163.0804443359375, 123.3778076171875, 165.00119018554688, 146.33189392089844, 76.0130615234375, 278.2159423828125, 112.83272552490234, 117.33250427246094, 219.61801147460938, 202.34776306152344, 66.94676208496094, 1170.4490966796875]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.9259202480316162, 0.08017925918102264, 0.19209080934524536, 0.06361857056617737, 0.007567114662379026, 0.017343170940876007, 0.0807785913348198, 0.014796692878007889, 0.007238495629280806, 0.10004221647977829, 0.05593104660511017, 0.025312207639217377, 0.02431764453649521, 0.07245571911334991, 0.1315336972475052, 0.12484225630760193, 0.18530867993831635, 0.16127800941467285, 0.08993591368198395, 0.2760690748691559, 0.1197303980588913, 0.10385147482156754, 0.21126008033752441, 0.21669329702854156, 0.06467969715595245, 1.0695146322250366]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.15089930379882974, -0.8290671241509645], "logp": -0.9799664279497943, "cache_id": 750, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 998.4506225585938, 78.0566635131836, 197.07781982421875, 67.62660217285156, 7.613514423370361, 17.8790340423584, 92.05768585205078, 15.205554962158203, 7.409041881561279, 97.7878189086914, 54.19824981689453, 25.320829391479492, 25.35968780517578, 75.88646697998047, 163.03302001953125, 123.41985321044922, 164.9969482421875, 146.29742431640625, 75.9881362915039, 278.145263671875, 112.75767517089844, 117.22711181640625, 219.6559600830078, 202.41775512695312, 66.92359924316406, 1170.4088134765625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 998.400634765625, 78.01586151123047, 197.13389587402344, 67.68624114990234, 7.727022647857666, 17.911195755004883, 92.10831451416016, 15.27455997467041, 7.607632637023926, 97.83941650390625, 54.20945358276367, 25.386960983276367, 25.40325355529785, 75.90375518798828, 163.0804443359375, 123.3778076171875, 165.00119018554688, 146.33189392089844, 76.0130615234375, 278.2159423828125, 112.83272552490234, 117.33250427246094, 219.61801147460938, 202.34776306152344, 66.94676208496094, 1170.4490966796875]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.9259202480316162, 0.08017925918102264, 0.19209080934524536, 0.06361857056617737, 0.007567114662379026, 0.017343170940876007, 0.0807785913348198, 0.014796692878007889, 0.007238495629280806, 0.10004221647977829, 0.05593104660511017, 0.025312207639217377, 0.02431764453649521, 0.07245571911334991, 0.1315336972475052, 0.12484225630760193, 0.18530867993831635, 0.16127800941467285, 0.08993591368198395, 0.2760690748691559, 0.1197303980588913, 0.10385147482156754, 0.21126008033752441, 0.21669329702854156, 0.06467969715595245, 1.0695146322250366]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 21.952182875849036, "first_log_odds": 22.78125, "later_log_odds": -0.8290671241509635, "first_differing_log_odds": 22.78125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-main-bridge.npz", "array_sha256": "b52e38ba4e08be2f2e6570e589546cb0bb1ea5339c0e53c23535c21dec4a79ee", "seconds": 0.3257967079989612, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-baseline", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 751, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.125036799043453], "logp": -20.125036799043453, "cache_id": 752, "trace": []}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.125036799043453, -1.117531782282751], "logp": -29.242568581326204, "cache_id": 753, "trace": []}, "log_odds": -9.11753178228275, "first_log_odds": -8.0, "later_log_odds": -1.1175317822827502, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-baseline.npz", "array_sha256": "bb9733d2f51a1d2f6f8a06c42042e22df48a0e5b541133cbe877f849d6d15ae8", "seconds": 1.12797479191795, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-noop", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.185251235961914, "match_error": -9.185251235961914, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 754, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.125036799043453], "logp": -20.125036799043453, "cache_id": 755, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.185251235961914, "match_error": -9.185251235961914, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.125036799043453, -1.117531782282751], "logp": -29.242568581326204, "cache_id": 756, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.185251235961914, "match_error": -9.185251235961914, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.11753178228275, "first_log_odds": -8.0, "later_log_odds": -1.1175317822827502, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-noop.npz", "array_sha256": "cb24017e589d30285169e6678ef3c221bd083c7050987db828c376ae4fac9ed6", "seconds": 1.128500375431031, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-j", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.187005043029785, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 0.23041875850740903, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 757, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.375040026370222], "logp": -20.375040026370222, "cache_id": 758, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.187005043029785, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 0.23041875850740903, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-27.500040026370222, -1.1932492688719991], "logp": -28.69328929524222, "cache_id": 759, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.187005043029785, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 0.23041875850740903, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.318249268871998, "first_log_odds": -7.125, "later_log_odds": -1.1932492688719982, "first_differing_log_odds": -7.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-j.npz", "array_sha256": "28ee8c010650c62da5938f7040303f71b4f1bc6f8a4816f4a6495e8f6e5897ea", "seconds": 1.228328375145793, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-opposite", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.186373710632324, "actual_norm": 9.184699058532715, "relative_norm": 0.049987450786195944, "target_actual_norm": 9.185251235961914, "match_error": -0.0005521774291992188, "scalar": 9.186372482450679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 760, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.62503375487609], "logp": -20.62503375487609, "cache_id": 761, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.186373710632324, "actual_norm": 9.184699058532715, "relative_norm": 0.049987450786195944, "target_actual_norm": 9.185251235961914, "match_error": -0.0005521774291992188, "scalar": 9.186372482450679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.75003375487609, -1.2810071407708028], "logp": -30.031040895646893, "cache_id": 762, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.186373710632324, "actual_norm": 9.184699058532715, "relative_norm": 0.049987450786195944, "target_actual_norm": 9.185251235961914, "match_error": -0.0005521774291992188, "scalar": 9.186372482450679, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.406007140770804, "first_log_odds": -8.125, "later_log_odds": -1.2810071407708037, "first_differing_log_odds": -8.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-opposite.npz", "array_sha256": "64d805f93fae84920d1f01cbc3cf9d3af9d2e1cfedd061dad3324a525ca8114a", "seconds": 1.2394099170342088, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-unrelated", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.1881685256958, "actual_norm": 9.185253143310547, "relative_norm": 0.04999046637607937, "target_actual_norm": 9.185251235961914, "match_error": 1.9073486328125e-06, "scalar": 9.188168228780341, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 763, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.37503118529458], "logp": -20.37503118529458, "cache_id": 764, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.1881685256958, "actual_norm": 9.185253143310547, "relative_norm": 0.04999046637607937, "target_actual_norm": 9.185251235961914, "match_error": 1.9073486328125e-06, "scalar": 9.188168228780341, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.06253118529458, -1.1202863898363642], "logp": -29.182817575130947, "cache_id": 765, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.1881685256958, "actual_norm": 9.185253143310547, "relative_norm": 0.04999046637607937, "target_actual_norm": 9.185251235961914, "match_error": 1.9073486328125e-06, "scalar": 9.188168228780341, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.807786389836366, "first_log_odds": -7.6875, "later_log_odds": -1.120286389836366, "first_differing_log_odds": -7.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-unrelated.npz", "array_sha256": "c65f2f3e3c570fc18bfb6b220db167069a1d52e8410645807f1cceffd7f672aa", "seconds": 1.1233148341998458, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-ordinary", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.18445873260498, "actual_norm": 9.185282707214355, "relative_norm": 0.04999062727674441, "target_actual_norm": 9.185251235961914, "match_error": 3.147125244140625e-05, "scalar": 9.18445738468813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 766, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.12505017442594], "logp": -20.12505017442594, "cache_id": 767, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.18445873260498, "actual_norm": 9.185282707214355, "relative_norm": 0.04999062727674441, "target_actual_norm": 9.185251235961914, "match_error": 3.147125244140625e-05, "scalar": 9.18445738468813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-27.75005017442594, -1.0481035346710468], "logp": -28.798153709096987, "cache_id": 768, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.18445873260498, "actual_norm": 9.185282707214355, "relative_norm": 0.04999062727674441, "target_actual_norm": 9.185251235961914, "match_error": 3.147125244140625e-05, "scalar": 9.18445738468813, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.673103534671046, "first_log_odds": -7.625, "later_log_odds": -1.0481035346710463, "first_differing_log_odds": -7.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-ordinary.npz", "array_sha256": "f84a701102a4cc2047c915608f4732736d61f0b0f31372ee0640cc6e908084af", "seconds": 1.1254323748871684, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-random0", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.183341026306152, "actual_norm": 9.185226440429688, "relative_norm": 0.049990321046446434, "target_actual_norm": 9.185251235961914, "match_error": -2.47955322265625e-05, "scalar": 9.183341613035736, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 769, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.500026822832442], "logp": -20.500026822832442, "cache_id": 770, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.183341026306152, "actual_norm": 9.185226440429688, "relative_norm": 0.049990321046446434, "target_actual_norm": 9.185251235961914, "match_error": -2.47955322265625e-05, "scalar": 9.183341613035736, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.562526822832442, -1.0480434340234583], "logp": -29.6105702568559, "cache_id": 771, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.183341026306152, "actual_norm": 9.185226440429688, "relative_norm": 0.049990321046446434, "target_actual_norm": 9.185251235961914, "match_error": -2.47955322265625e-05, "scalar": 9.183341613035736, "random_seed": 20272208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.110543434023459, "first_log_odds": -8.0625, "later_log_odds": -1.0480434340234588, "first_differing_log_odds": -8.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-random0.npz", "array_sha256": "1c7a01d7d52708131eb1a81deac648088d8cc66c1b1882c48d306ffc4c2378ae", "seconds": 1.128384916111827, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-random1", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.185548782348633, "actual_norm": 9.185234069824219, "relative_norm": 0.049990362569198706, "target_actual_norm": 9.185251235961914, "match_error": -1.71661376953125e-05, "scalar": 9.185549067060492, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 772, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-20.250036885067907], "logp": -20.250036885067907, "cache_id": 773, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.185548782348633, "actual_norm": 9.185234069824219, "relative_norm": 0.049990362569198706, "target_actual_norm": 9.185251235961914, "match_error": -1.71661376953125e-05, "scalar": 9.185549067060492, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.125036885067907, -1.1132651953324832], "logp": -29.23830208040039, "cache_id": 774, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.185548782348633, "actual_norm": 9.185234069824219, "relative_norm": 0.049990362569198706, "target_actual_norm": 9.185251235961914, "match_error": -1.71661376953125e-05, "scalar": 9.185549067060492, "random_seed": 20272209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.988265195332485, "first_log_odds": -7.875, "later_log_odds": -1.1132651953324846, "first_differing_log_odds": -7.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-random1.npz", "array_sha256": "00395ddb73d7486b55a219fbbe6743adbf161a037e63dccc5afd3df15ac5a09d", "seconds": 1.125072166789323, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-random2", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.162361145019531, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 9.162359850127814, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 775, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-21.250024665173104], "logp": -21.250024665173104, "cache_id": 776, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.162361145019531, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 9.162359850127814, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-28.812524665173104, -1.2007173747239204], "logp": -30.013242039897023, "cache_id": 777, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 183.74009704589844, "strength_one_norm": 39.870906829833984, "target_shrink_scale": 0.23041875850740903, "requested_norm": 9.162361145019531, "actual_norm": 9.185251235961914, "relative_norm": 0.04999045599539131, "target_actual_norm": 9.185251235961914, "match_error": 0.0, "scalar": 9.162359850127814, "random_seed": 20272210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.763217374723919, "first_log_odds": -7.5625, "later_log_odds": -1.2007173747239186, "first_differing_log_odds": -7.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-random2.npz", "array_sha256": "a53198eb78cfe8cec84d56440c780151f4938f5fd52619297254b3178b268072", "seconds": 1.124219042249024, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-13-paraphrase-bridge", "case_id": "eval-13", "city": "Istanbul", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.<|im_end|>", "output_text": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.", "output_ids": [785, 11413, 1483, 304, 53368, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 15881, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 957.3621826171875, 45.76362228393555, 152.45684814453125, 269.6009521484375, 38.23329162597656, 2.5797348022460938, 99.99894714355469, 24.13886260986328, 16.574033737182617, 97.28970336914062, 40.9549446105957, 6.525335311889648, 68.85560607910156, 166.9508819580078, 119.54804229736328, 90.51144409179688, 294.8965759277344, 140.3841552734375, 169.5746307373047, 237.08973693847656, 302.1048889160156, 65.87538146972656, 799.8738403320312]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 957.275146484375, 45.76055908203125, 152.44061279296875, 269.64788818359375, 38.21430206298828, 2.648639678955078, 100.01930236816406, 24.209068298339844, 16.61945343017578, 97.3067626953125, 40.97323989868164, 6.637714385986328, 68.87264251708984, 166.92955017089844, 119.57241821289062, 90.51696014404297, 294.9260559082031, 140.3407440185547, 169.55052185058594, 237.06130981445312, 302.083984375, 65.91497802734375, 799.905517578125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.8976446986198425, 0.0439886637032032, 0.1390342116355896, 0.24690435826778412, 0.03496422991156578, 0.002578756073489785, 0.085603728890419, 0.022883011028170586, 0.015680329874157906, 0.0984165221452713, 0.04194226488471031, 0.006497348193079233, 0.06520608812570572, 0.17377716302871704, 0.13494521379470825, 0.10433320701122284, 0.2879727780818939, 0.1495279222726822, 0.1490151584148407, 0.22402061522006989, 0.31966808438301086, 0.06351971626281738, 0.8096367716789246]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 778, "cached": true, "scoring": {"normalized": "The currency used in Istanbul is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [{"text": "Turkish", "start": 37, "end": 44}], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-25.185791504442165], "logp": -25.185791504442165, "cache_id": 779, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 957.3621826171875, 45.76362228393555, 152.45684814453125, 269.6009521484375, 38.23329162597656, 2.5797348022460938, 99.99894714355469, 24.13886260986328, 16.574033737182617, 97.28970336914062, 40.9549446105957, 6.525335311889648, 68.85560607910156, 166.9508819580078, 119.54804229736328, 90.51144409179688, 294.8965759277344, 140.3841552734375, 169.5746307373047, 237.08973693847656, 302.1048889160156, 65.87538146972656, 799.8738403320312]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 957.275146484375, 45.76055908203125, 152.44061279296875, 269.64788818359375, 38.21430206298828, 2.648639678955078, 100.01930236816406, 24.209068298339844, 16.61945343017578, 97.3067626953125, 40.97323989868164, 6.637714385986328, 68.87264251708984, 166.92955017089844, 119.57241821289062, 90.51696014404297, 294.9260559082031, 140.3407440185547, 169.55052185058594, 237.06130981445312, 302.083984375, 65.91497802734375, 799.905517578125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.8976446986198425, 0.0439886637032032, 0.1390342116355896, 0.24690435826778412, 0.03496422991156578, 0.002578756073489785, 0.085603728890419, 0.022883011028170586, 0.015680329874157906, 0.0984165221452713, 0.04194226488471031, 0.006497348193079233, 0.06520608812570572, 0.17377716302871704, 0.13494521379470825, 0.10433320701122284, 0.2879727780818939, 0.1495279222726822, 0.1490151584148407, 0.22402061522006989, 0.31966808438301086, 0.06351971626281738, 0.8096367716789246]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.9982915044421627, -0.2785289657081577], "logp": -1.2768204701503205, "cache_id": 780, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 957.3621826171875, 45.76362228393555, 152.45684814453125, 269.6009521484375, 38.23329162597656, 2.5797348022460938, 99.99894714355469, 24.13886260986328, 16.574033737182617, 97.28970336914062, 40.9549446105957, 6.525335311889648, 68.85560607910156, 166.9508819580078, 119.54804229736328, 90.51144409179688, 294.8965759277344, 140.3841552734375, 169.5746307373047, 237.08973693847656, 302.1048889160156, 65.87538146972656, 799.8738403320312]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 957.275146484375, 45.76055908203125, 152.44061279296875, 269.64788818359375, 38.21430206298828, 2.648639678955078, 100.01930236816406, 24.209068298339844, 16.61945343017578, 97.3067626953125, 40.97323989868164, 6.637714385986328, 68.87264251708984, 166.92955017089844, 119.57241821289062, 90.51696014404297, 294.9260559082031, 140.3407440185547, 169.55052185058594, 237.06130981445312, 302.083984375, 65.91497802734375, 799.905517578125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.8976446986198425, 0.0439886637032032, 0.1390342116355896, 0.24690435826778412, 0.03496422991156578, 0.002578756073489785, 0.085603728890419, 0.022883011028170586, 0.015680329874157906, 0.0984165221452713, 0.04194226488471031, 0.006497348193079233, 0.06520608812570572, 0.17377716302871704, 0.13494521379470825, 0.10433320701122284, 0.2879727780818939, 0.1495279222726822, 0.1490151584148407, 0.22402061522006989, 0.31966808438301086, 0.06351971626281738, 0.8096367716789246]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 23.908971034291845, "first_log_odds": 24.1875, "later_log_odds": -0.2785289657081549, "first_differing_log_odds": 24.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-13-paraphrase-bridge.npz", "array_sha256": "ecb10108c78fe8499a73a2c5161c30f32c22a49669f6f42c93876d11622b8649", "seconds": 1.1263193748891354, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-baseline", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 781, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.225890163739749], "logp": -8.225890163739749, "cache_id": 782, "trace": []}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.97589016373975, -0.8575668732993766], "logp": -17.833457037039125, "cache_id": 783, "trace": []}, "log_odds": -9.607566873299376, "first_log_odds": -8.75, "later_log_odds": -0.8575668732993762, "first_differing_log_odds": -8.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-baseline.npz", "array_sha256": "3cc543f1963aa957a5409117d01c4caa68524396c87f0662adb2a675dafdd268", "seconds": 0.3277432909235358, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-noop", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.187039375305176, "match_error": -9.187039375305176, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 784, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.225890163739749], "logp": -8.225890163739749, "cache_id": 785, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.187039375305176, "match_error": -9.187039375305176, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.97589016373975, -0.8575668732993766], "logp": -17.833457037039125, "cache_id": 786, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.187039375305176, "match_error": -9.187039375305176, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.607566873299376, "first_log_odds": -8.75, "later_log_odds": -0.8575668732993762, "first_differing_log_odds": -8.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-noop.npz", "array_sha256": "4cfc9e924a51592d6e2b145324ba77a14663a47de4b6b6939e1d4eb564c502be", "seconds": 0.2912279167212546, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-j", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.181918144226074, "actual_norm": 9.187039375305176, "relative_norm": 0.049999793425937196, "target_actual_norm": 9.187039375305176, "match_error": 0.0, "scalar": 0.6096213757254029, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 787, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.002796796734707], "logp": -8.002796796734707, "cache_id": 788, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.181918144226074, "actual_norm": 9.187039375305176, "relative_norm": 0.049999793425937196, "target_actual_norm": 9.187039375305176, "match_error": 0.0, "scalar": 0.6096213757254029, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.627796796734707, -0.8670519038333645], "logp": -17.49484870056807, "cache_id": 789, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.181918144226074, "actual_norm": 9.187039375305176, "relative_norm": 0.049999793425937196, "target_actual_norm": 9.187039375305176, "match_error": 0.0, "scalar": 0.6096213757254029, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.492051903833364, "first_log_odds": -8.625, "later_log_odds": -0.8670519038333637, "first_differing_log_odds": -8.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-j.npz", "array_sha256": "94651e6a01901c2d60c2cc324c51a5720652026e803e9d9016d2c8dfa948df42", "seconds": 0.29133524978533387, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-opposite", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.17905044555664, "actual_norm": 9.186716079711914, "relative_norm": 0.04999803391319123, "target_actual_norm": 9.187039375305176, "match_error": -0.00032329559326171875, "scalar": 9.179048938836786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 790, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.11356402203541], "logp": -8.11356402203541, "cache_id": 791, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.17905044555664, "actual_norm": 9.186716079711914, "relative_norm": 0.04999803391319123, "target_actual_norm": 9.187039375305176, "match_error": -0.00032329559326171875, "scalar": 9.179048938836786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-17.11356402203541, -1.10626184363043], "logp": -18.21982586566584, "cache_id": 792, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.17905044555664, "actual_norm": 9.186716079711914, "relative_norm": 0.04999803391319123, "target_actual_norm": 9.187039375305176, "match_error": -0.00032329559326171875, "scalar": 9.179048938836786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.10626184363043, "first_log_odds": -8.999999999999998, "later_log_odds": -1.1062618436304312, "first_differing_log_odds": -8.999999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-opposite.npz", "array_sha256": "4d37047dadecc8a8e03f1ed5634a42b3cacd335d708d78ebbd16e64e18c03bf7", "seconds": 0.29217704199254513, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-unrelated", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180625915527344, "actual_norm": 9.187003135681152, "relative_norm": 0.04999959619441995, "target_actual_norm": 9.187039375305176, "match_error": -3.62396240234375e-05, "scalar": 9.180625998666073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 793, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-7.976059803799653], "logp": -7.976059803799653, "cache_id": 794, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180625915527344, "actual_norm": 9.187003135681152, "relative_norm": 0.04999959619441995, "target_actual_norm": 9.187039375305176, "match_error": -3.62396240234375e-05, "scalar": 9.180625998666073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.851059803799654, -0.849878393040895], "logp": -17.70093819684055, "cache_id": 795, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180625915527344, "actual_norm": 9.187003135681152, "relative_norm": 0.04999959619441995, "target_actual_norm": 9.187039375305176, "match_error": -3.62396240234375e-05, "scalar": 9.180625998666073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.724878393040896, "first_log_odds": -8.875, "later_log_odds": -0.8498783930408962, "first_differing_log_odds": -8.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-unrelated.npz", "array_sha256": "1a1c5c81795ea88e001408be1d9b488a8c22a35ecc9015cd8656ff8856babeae", "seconds": 0.2920479169115424, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-ordinary", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.190473556518555, "actual_norm": 9.18700885772705, "relative_norm": 0.049999627336238466, "target_actual_norm": 9.187039375305176, "match_error": -3.0517578125e-05, "scalar": 9.190473861155624, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 796, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.22590266927978], "logp": -8.22590266927978, "cache_id": 797, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.190473556518555, "actual_norm": 9.18700885772705, "relative_norm": 0.049999627336238466, "target_actual_norm": 9.187039375305176, "match_error": -3.0517578125e-05, "scalar": 9.190473861155624, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.97590266927978, -0.9724150214662517], "logp": -17.94831769074603, "cache_id": 798, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.190473556518555, "actual_norm": 9.18700885772705, "relative_norm": 0.049999627336238466, "target_actual_norm": 9.187039375305176, "match_error": -3.0517578125e-05, "scalar": 9.190473861155624, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.72241502146625, "first_log_odds": -8.75, "later_log_odds": -0.9724150214662508, "first_differing_log_odds": -8.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-ordinary.npz", "array_sha256": "c2fe91417f99af9898ae2d7ef1993372f02ccc845209acbe1c358875c2dd41bc", "seconds": 0.2926100417971611, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-random0", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.171969413757324, "actual_norm": 9.187054634094238, "relative_norm": 0.04999987647078657, "target_actual_norm": 9.187039375305176, "match_error": 1.52587890625e-05, "scalar": 9.171969692491984, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 799, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-7.877853966123637], "logp": -7.877853966123637, "cache_id": 800, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.171969413757324, "actual_norm": 9.187054634094238, "relative_norm": 0.04999987647078657, "target_actual_norm": 9.187039375305176, "match_error": 1.52587890625e-05, "scalar": 9.171969692491984, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.627853966123638, -0.9811189390864962], "logp": -17.608972905210134, "cache_id": 801, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.171969413757324, "actual_norm": 9.187054634094238, "relative_norm": 0.04999987647078657, "target_actual_norm": 9.187039375305176, "match_error": 1.52587890625e-05, "scalar": 9.171969692491984, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.731118939086496, "first_log_odds": -8.75, "later_log_odds": -0.9811189390864961, "first_differing_log_odds": -8.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-random0.npz", "array_sha256": "e888e2b4a66b9daccdef00001dc37df38bae6a21f59002d0509a6db4593184a8", "seconds": 0.29137158300727606, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-random1", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180171966552734, "actual_norm": 9.186873435974121, "relative_norm": 0.04999889031320033, "target_actual_norm": 9.187039375305176, "match_error": -0.0001659393310546875, "scalar": 9.18017040360428, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 802, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.127762833160503], "logp": -8.127762833160503, "cache_id": 803, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180171966552734, "actual_norm": 9.186873435974121, "relative_norm": 0.04999889031320033, "target_actual_norm": 9.187039375305176, "match_error": -0.0001659393310546875, "scalar": 9.18017040360428, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-16.752762833160503, -0.9802045336187423], "logp": -17.732967366779246, "cache_id": 804, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.180171966552734, "actual_norm": 9.186873435974121, "relative_norm": 0.04999889031320033, "target_actual_norm": 9.187039375305176, "match_error": -0.0001659393310546875, "scalar": 9.18017040360428, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.605204533618743, "first_log_odds": -8.625, "later_log_odds": -0.9802045336187426, "first_differing_log_odds": -8.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-random1.npz", "array_sha256": "f838e5a445b8b8bf4b9580a9b5ad1707804f80d24f9c7fdfa25e49586354f843", "seconds": 0.29188091680407524, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-random2", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.182729721069336, "actual_norm": 9.187049865722656, "relative_norm": 0.04999985051927114, "target_actual_norm": 9.187039375305176, "match_error": 1.049041748046875e-05, "scalar": 9.182728745105123, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 805, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-8.2259023394714], "logp": -8.2259023394714, "cache_id": 806, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.182729721069336, "actual_norm": 9.187049865722656, "relative_norm": 0.04999985051927114, "target_actual_norm": 9.187039375305176, "match_error": 1.049041748046875e-05, "scalar": 9.182728745105123, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-17.1009023394714, -0.9614459119121925], "logp": -18.062348251383593, "cache_id": 807, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 183.74154663085938, "strength_one_norm": 15.061673164367676, "target_shrink_scale": 0.6096213757254029, "requested_norm": 9.182729721069336, "actual_norm": 9.187049865722656, "relative_norm": 0.04999985051927114, "target_actual_norm": 9.187039375305176, "match_error": 1.049041748046875e-05, "scalar": 9.182728745105123, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.836445911912193, "first_log_odds": -8.875, "later_log_odds": -0.9614459119121932, "first_differing_log_odds": -8.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-random2.npz", "array_sha256": "d0a881792c8e1945529043f13b88c9f16ac4be19d9f6c4286df1cc4db0ca22ed", "seconds": 0.2910070000216365, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-main-bridge", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 74.058349609375, 665.66064453125, 762.9971923828125, 104.76023864746094, 189.0593719482422, 71.06038665771484, 25.098817825317383, 14.59970474243164, 85.41602325439453, 22.331775665283203, 0.4198477268218994, 100.79222106933594, 52.71921920776367, 19.10135841369629, 16.707897186279297, 74.23921966552734, 162.1761932373047, 128.95697021484375, 198.77159118652344, 126.2799301147461, 75.89244079589844, 276.5164489746094, 112.10315704345703, 112.25285339355469, 200.2887420654297, 177.300537109375, 73.7433090209961, 928.6656494140625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 74.1065444946289, 665.6290893554688, 763.0304565429688, 104.7413558959961, 189.09413146972656, 71.05946350097656, 25.17131996154785, 14.658409118652344, 85.39549255371094, 22.443944931030273, 0.2520831525325775, 100.78804016113281, 52.770904541015625, 19.095855712890625, 16.76011848449707, 74.27515411376953, 162.21728515625, 128.99232482910156, 198.77871704101562, 126.29428100585938, 75.92363739013672, 276.50146484375, 112.12732696533203, 112.28427124023438, 200.35406494140625, 177.30657958984375, 73.69817352294922, 928.5235595703125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.0569150447845459, 0.7063109278678894, 0.7299904823303223, 0.10772394388914108, 0.18593192100524902, 0.06734373420476913, 0.024695873260498047, 0.014295932836830616, 0.07500102370977402, 0.022156020626425743, 0.00024074011889752, 0.10261426120996475, 0.05462769418954849, 0.019478721544146538, 0.0160415880382061, 0.07047621160745621, 0.12996815145015717, 0.12930327653884888, 0.22326499223709106, 0.14039506018161774, 0.0913994163274765, 0.27593764662742615, 0.11902930587530136, 0.09943518787622452, 0.19451062381267548, 0.19233249127864838, 0.07105419784784317, 0.9003353118896484]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 808, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-23.262785204732918], "logp": -23.262785204732918, "cache_id": 809, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 74.058349609375, 665.66064453125, 762.9971923828125, 104.76023864746094, 189.0593719482422, 71.06038665771484, 25.098817825317383, 14.59970474243164, 85.41602325439453, 22.331775665283203, 0.4198477268218994, 100.79222106933594, 52.71921920776367, 19.10135841369629, 16.707897186279297, 74.23921966552734, 162.1761932373047, 128.95697021484375, 198.77159118652344, 126.2799301147461, 75.89244079589844, 276.5164489746094, 112.10315704345703, 112.25285339355469, 200.2887420654297, 177.300537109375, 73.7433090209961, 928.6656494140625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 74.1065444946289, 665.6290893554688, 763.0304565429688, 104.7413558959961, 189.09413146972656, 71.05946350097656, 25.17131996154785, 14.658409118652344, 85.39549255371094, 22.443944931030273, 0.2520831525325775, 100.78804016113281, 52.770904541015625, 19.095855712890625, 16.76011848449707, 74.27515411376953, 162.21728515625, 128.99232482910156, 198.77871704101562, 126.29428100585938, 75.92363739013672, 276.50146484375, 112.12732696533203, 112.28427124023438, 200.35406494140625, 177.30657958984375, 73.69817352294922, 928.5235595703125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.0569150447845459, 0.7063109278678894, 0.7299904823303223, 0.10772394388914108, 0.18593192100524902, 0.06734373420476913, 0.024695873260498047, 0.014295932836830616, 0.07500102370977402, 0.022156020626425743, 0.00024074011889752, 0.10261426120996475, 0.05462769418954849, 0.019478721544146538, 0.0160415880382061, 0.07047621160745621, 0.12996815145015717, 0.12930327653884888, 0.22326499223709106, 0.14039506018161774, 0.0913994163274765, 0.27593764662742615, 0.11902930587530136, 0.09943518787622452, 0.19451062381267548, 0.19233249127864838, 0.07105419784784317, 0.9003353118896484]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.012785204732918776, -0.23110624264391927], "logp": -0.24389144737683804, "cache_id": 810, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 127.5361557006836, 43.84097671508789, 143.09288024902344, 99.43091583251953, 231.9453125, 74.058349609375, 665.66064453125, 762.9971923828125, 104.76023864746094, 189.0593719482422, 71.06038665771484, 25.098817825317383, 14.59970474243164, 85.41602325439453, 22.331775665283203, 0.4198477268218994, 100.79222106933594, 52.71921920776367, 19.10135841369629, 16.707897186279297, 74.23921966552734, 162.1761932373047, 128.95697021484375, 198.77159118652344, 126.2799301147461, 75.89244079589844, 276.5164489746094, 112.10315704345703, 112.25285339355469, 200.2887420654297, 177.300537109375, 73.7433090209961, 928.6656494140625]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 127.50175476074219, 43.9051513671875, 143.12841796875, 99.42461395263672, 231.93446350097656, 74.1065444946289, 665.6290893554688, 763.0304565429688, 104.7413558959961, 189.09413146972656, 71.05946350097656, 25.17131996154785, 14.658409118652344, 85.39549255371094, 22.443944931030273, 0.2520831525325775, 100.78804016113281, 52.770904541015625, 19.095855712890625, 16.76011848449707, 74.27515411376953, 162.21728515625, 128.99232482910156, 198.77871704101562, 126.29428100585938, 75.92363739013672, 276.50146484375, 112.12732696533203, 112.28427124023438, 200.35406494140625, 177.30657958984375, 73.69817352294922, 928.5235595703125]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.11704213172197342, 0.03852156177163124, 0.11644256114959717, 0.08509954065084457, 0.2043808251619339, 0.0569150447845459, 0.7063109278678894, 0.7299904823303223, 0.10772394388914108, 0.18593192100524902, 0.06734373420476913, 0.024695873260498047, 0.014295932836830616, 0.07500102370977402, 0.022156020626425743, 0.00024074011889752, 0.10261426120996475, 0.05462769418954849, 0.019478721544146538, 0.0160415880382061, 0.07047621160745621, 0.12996815145015717, 0.12930327653884888, 0.22326499223709106, 0.14039506018161774, 0.0913994163274765, 0.27593764662742615, 0.11902930587530136, 0.09943518787622452, 0.19451062381267548, 0.19233249127864838, 0.07105419784784317, 0.9003353118896484]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 23.01889375735608, "first_log_odds": 23.25, "later_log_odds": -0.23110624264392143, "first_differing_log_odds": 23.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-main-bridge.npz", "array_sha256": "7f149e23be340d898992aad19a1711672a446fc87257439593053374e19ceeb8", "seconds": 0.3274483340792358, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-baseline", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 811, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.375011059218537], "logp": -26.375011059218537, "cache_id": 812, "trace": []}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.500011059218537, -1.2858951484855603], "logp": -32.785906207704095, "cache_id": 813, "trace": []}, "log_odds": -6.410895148485558, "first_log_odds": -5.125, "later_log_odds": -1.285895148485558, "first_differing_log_odds": -5.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-baseline.npz", "array_sha256": "aa0868bc265960c45d6b802f78ad008287ff3dd4f84a7245a7fb95002f15e384", "seconds": 1.4621811658143997, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-noop", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.235921859741211, "match_error": -9.235921859741211, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 814, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.375011059218537], "logp": -26.375011059218537, "cache_id": 815, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.235921859741211, "match_error": -9.235921859741211, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.500011059218537, -1.2858951484855603], "logp": -32.785906207704095, "cache_id": 816, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.235921859741211, "match_error": -9.235921859741211, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.410895148485558, "first_log_odds": -5.125, "later_log_odds": -1.285895148485558, "first_differing_log_odds": -5.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-noop.npz", "array_sha256": "7a54fef2d0c2d5e0261a53247d15ab6d9382053b0bfbf20140e7fd3c93829daf", "seconds": 1.4371012919582427, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-j", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.237606048583984, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 0.6225924539958768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 817, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.500010910643425], "logp": -26.500010910643425, "cache_id": 818, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.237606048583984, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 0.6225924539958768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.375010910643425, -1.4005323859493903], "logp": -32.775543296592815, "cache_id": 819, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.237606048583984, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 0.6225924539958768, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.27553238594939, "first_log_odds": -4.875, "later_log_odds": -1.40053238594939, "first_differing_log_odds": -4.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-j.npz", "array_sha256": "19219f0a652076b4042df236053ae18e0cd9532f4786d2a5359c5355fb62d929", "seconds": 1.4655316667631269, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-opposite", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.<|im_end|>", "output_text": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 374, 279, 19461, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 54738, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.228976249694824, "actual_norm": 9.23592472076416, "relative_norm": 0.04999089954794034, "target_actual_norm": 9.235921859741211, "match_error": 2.86102294921875e-06, "scalar": 9.228976703275293, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 820, "cached": true, "scoring": {"normalized": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.", "bare_format": false, "malformed": true, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-25.87501814343167], "logp": -25.87501814343167, "cache_id": 821, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.228976249694824, "actual_norm": 9.23592472076416, "relative_norm": 0.04999089954794034, "target_actual_norm": 9.235921859741211, "match_error": 2.86102294921875e-06, "scalar": 9.228976703275293, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.25001814343167, -1.2534684432583307], "logp": -32.50348658669, "cache_id": 822, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.228976249694824, "actual_norm": 9.23592472076416, "relative_norm": 0.04999089954794034, "target_actual_norm": 9.235921859741211, "match_error": 2.86102294921875e-06, "scalar": 9.228976703275293, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.628468443258331, "first_log_odds": -5.375, "later_log_odds": -1.2534684432583312, "first_differing_log_odds": -5.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-opposite.npz", "array_sha256": "e42bfee2c40a6b763792a8c692bb9b216d401737966c15934f9aba5f66164886", "seconds": 1.1579439169727266, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-unrelated", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.23620319366455, "actual_norm": 9.235930442810059, "relative_norm": 0.04999093051941643, "target_actual_norm": 9.235921859741211, "match_error": 8.58306884765625e-06, "scalar": 9.236203717708122, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 823, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.250011021229284], "logp": -26.250011021229284, "cache_id": 824, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.23620319366455, "actual_norm": 9.235930442810059, "relative_norm": 0.04999093051941643, "target_actual_norm": 9.235921859741211, "match_error": 8.58306884765625e-06, "scalar": 9.236203717708122, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.562511021229284, -1.1983863743079168], "logp": -32.7608973955372, "cache_id": 825, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.23620319366455, "actual_norm": 9.235930442810059, "relative_norm": 0.04999093051941643, "target_actual_norm": 9.235921859741211, "match_error": 8.58306884765625e-06, "scalar": 9.236203717708122, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.5108863743079155, "first_log_odds": -5.3125, "later_log_odds": -1.1983863743079155, "first_differing_log_odds": -5.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-unrelated.npz", "array_sha256": "8101d8b54632242cfc5aa60380552afa7a81eb9d6e56900a8b031dd1b1d3524a", "seconds": 1.491725584026426, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-ordinary", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like all of Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.<|im_end|>", "output_text": "The currency used in Antalya, like all of Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 678, 315, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 15881, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.243535995483398, "actual_norm": 9.235917091369629, "relative_norm": 0.049990858252638874, "target_actual_norm": 9.235921859741211, "match_error": -4.76837158203125e-06, "scalar": 9.243536428878542, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 826, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like all of Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [{"text": "Turkey", "start": 42, "end": 48}, {"text": "Turkish", "start": 57, "end": 64}], "country_before_code": false, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.37500869501533], "logp": -26.37500869501533, "cache_id": 827, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.243535995483398, "actual_norm": 9.235917091369629, "relative_norm": 0.049990858252638874, "target_actual_norm": 9.235921859741211, "match_error": -4.76837158203125e-06, "scalar": 9.243536428878542, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.56250869501533, -1.2272578695822745], "logp": -32.789766564597606, "cache_id": 828, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.243535995483398, "actual_norm": 9.235917091369629, "relative_norm": 0.049990858252638874, "target_actual_norm": 9.235921859741211, "match_error": -4.76837158203125e-06, "scalar": 9.243536428878542, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.4147578695822745, "first_log_odds": -5.1875, "later_log_odds": -1.2272578695822745, "first_differing_log_odds": -5.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-ordinary.npz", "array_sha256": "2cca3740e10861e4e2da15982192799f4a65f81b9ea60e29f2d90094f2291ace", "seconds": 1.4102246253751218, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-random0", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like all of Turkey, is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.<|im_end|>", "output_text": "The currency used in Antalya, like all of Turkey, is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 678, 315, 17009, 11, 374, 279, 23734, 326, 8832, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 15881, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.224125862121582, "actual_norm": 9.236062049865723, "relative_norm": 0.04999164286336665, "target_actual_norm": 9.235921859741211, "match_error": 0.00014019012451171875, "scalar": 9.22412566342291, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 829, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like all of Turkey, is the Turkish lira, and its three-letter ISO 4217 code is **TRL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [{"text": "Turkey", "start": 42, "end": 48}, {"text": "Turkish", "start": 57, "end": 64}], "country_before_code": false, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.250011215948728], "logp": -26.250011215948728, "cache_id": 830, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.224125862121582, "actual_norm": 9.236062049865723, "relative_norm": 0.04999164286336665, "target_actual_norm": 9.235921859741211, "match_error": 0.00014019012451171875, "scalar": 9.22412566342291, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.437511215948728, -1.2847772994385562], "logp": -32.722288515387284, "cache_id": 831, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.224125862121582, "actual_norm": 9.236062049865723, "relative_norm": 0.04999164286336665, "target_actual_norm": 9.235921859741211, "match_error": 0.00014019012451171875, "scalar": 9.22412566342291, "random_seed": 20272308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.472277299438556, "first_log_odds": -5.1875, "later_log_odds": -1.2847772994385558, "first_differing_log_odds": -5.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-random0.npz", "array_sha256": "9a5ebd24aa4c7bc492b6a6d7ca17af034739c865044ba2603985df5f07b363c6", "seconds": 1.436701999977231, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-random1", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 26201, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.225204467773438, "actual_norm": 9.235926628112793, "relative_norm": 0.0499909098717657, "target_actual_norm": 9.235921859741211, "match_error": 4.76837158203125e-06, "scalar": 9.225202448937125, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 832, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRY**.", "bare_format": false, "malformed": true, "first_code": "TRY", "all_codes": ["TRY"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.750008535497987], "logp": -26.750008535497987, "cache_id": 833, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.225204467773438, "actual_norm": 9.235926628112793, "relative_norm": 0.0499909098717657, "target_actual_norm": 9.235921859741211, "match_error": 4.76837158203125e-06, "scalar": 9.225202448937125, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.750008535497987, -1.2848241007481698], "logp": -33.03483263624616, "cache_id": 834, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.225204467773438, "actual_norm": 9.235926628112793, "relative_norm": 0.0499909098717657, "target_actual_norm": 9.235921859741211, "match_error": 4.76837158203125e-06, "scalar": 9.225202448937125, "random_seed": 20272309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.284824100748171, "first_log_odds": -5.0, "later_log_odds": -1.284824100748171, "first_differing_log_odds": -5.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-random1.npz", "array_sha256": "6fe0e5e54dba8834c72e821ca7e89512fe4cd04fab08d2881f332250db129cdf", "seconds": 1.4796936251223087, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-random2", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.<|im_end|>", "output_text": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 11, 1075, 279, 11413, 1483, 6814, 17009, 11, 374, 279, 23734, 326, 8832, 13, 11445, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 15881, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.232538223266602, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 9.232539564138278, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 835, "cached": true, "scoring": {"normalized": "The currency used in Antalya, like the currency used throughout Turkey, is the Turkish lira. Its three-letter ISO 4217 code is **TRL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [{"text": "Turkey", "start": 64, "end": 70}, {"text": "Turkish", "start": 79, "end": 86}], "country_before_code": false, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-26.375013909414907], "logp": -26.375013909414907, "cache_id": 836, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.232538223266602, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 9.232539564138278, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-31.437513909414907, -1.3599165937510862], "logp": -32.79743050316599, "cache_id": 837, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 184.7521209716797, "strength_one_norm": 14.837324142456055, "target_shrink_scale": 0.6225924539958768, "requested_norm": 9.232538223266602, "actual_norm": 9.235921859741211, "relative_norm": 0.049990884062202284, "target_actual_norm": 9.235921859741211, "match_error": 0.0, "scalar": 9.232539564138278, "random_seed": 20272310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.422416593751084, "first_log_odds": -5.0625, "later_log_odds": -1.359916593751084, "first_differing_log_odds": -5.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-random2.npz", "array_sha256": "3f5675b3a1a8fadba9b8cd84d66659292e155d2c06384e3e09355bd839501879", "seconds": 1.4413142502307892, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-14-paraphrase-bridge", "case_id": "eval-14", "city": "Antalya", "country": "Turkey", "code": "TRY", "target_country": "Thailand", "target_code": "THB", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.<|im_end|>", "output_text": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.", "output_ids": [785, 11413, 1483, 304, 6755, 5774, 64, 374, 279, 19461, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 54738, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 62.839900970458984, 669.7349243164062, 694.6262817382812, 93.17864227294922, 135.43772888183594, 236.87692260742188, 54.92435836791992, 22.82176399230957, 94.26825714111328, 25.40935516357422, 0.4272097945213318, 105.65047454833984, 45.25754165649414, 5.396683216094971, 78.68418884277344, 180.51022338867188, 138.62718200683594, 105.86917114257812, 292.4052734375, 132.739013671875, 173.25091552734375, 249.2261505126953, 316.7022399902344, 70.60674285888672, 597.735595703125]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 62.85100555419922, 669.785888671875, 694.6476440429688, 93.1737060546875, 135.44766235351562, 236.87062072753906, 54.96247482299805, 22.90073585510254, 94.2580795288086, 25.52640151977539, 0.24288122355937958, 105.6361312866211, 45.29419708251953, 5.501523017883301, 78.71949768066406, 180.50009155273438, 138.7248077392578, 105.84021759033203, 292.4077453613281, 132.728759765625, 173.36131286621094, 249.2428436279297, 316.71759033203125, 70.60920715332031, 597.7246704101562]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.04824884608387947, 0.7124463319778442, 0.6619107127189636, 0.09382154047489166, 0.12562134861946106, 0.22796857357025146, 0.051236074417829514, 0.022322192788124084, 0.08145623654127121, 0.024263404309749603, 0.00022821231686975807, 0.10714813321828842, 0.04663172364234924, 0.005505691282451153, 0.07477705925703049, 0.1910715401172638, 0.15616773068904877, 0.1230965107679367, 0.286618709564209, 0.14264000952243805, 0.1529824584722519, 0.23666103184223175, 0.33499470353126526, 0.06775948405265808, 0.6257417798042297]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 838, "cached": true, "scoring": {"normalized": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **EUR**.", "bare_format": false, "malformed": true, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Antalya is the Euro, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [26201], "tokens": ["TRY"], "log_probs": [-31.281759047476793], "logp": -31.281759047476793, "cache_id": 839, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 62.839900970458984, 669.7349243164062, 694.6262817382812, 93.17864227294922, 135.43772888183594, 236.87692260742188, 54.92435836791992, 22.82176399230957, 94.26825714111328, 25.40935516357422, 0.4272097945213318, 105.65047454833984, 45.25754165649414, 5.396683216094971, 78.68418884277344, 180.51022338867188, 138.62718200683594, 105.86917114257812, 292.4052734375, 132.739013671875, 173.25091552734375, 249.2261505126953, 316.7022399902344, 70.60674285888672, 597.735595703125]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 62.85100555419922, 669.785888671875, 694.6476440429688, 93.1737060546875, 135.44766235351562, 236.87062072753906, 54.96247482299805, 22.90073585510254, 94.2580795288086, 25.52640151977539, 0.24288122355937958, 105.6361312866211, 45.29419708251953, 5.501523017883301, 78.71949768066406, 180.50009155273438, 138.7248077392578, 105.84021759033203, 292.4077453613281, 132.728759765625, 173.36131286621094, 249.2428436279297, 316.71759033203125, 70.60920715332031, 597.7246704101562]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.04824884608387947, 0.7124463319778442, 0.6619107127189636, 0.09382154047489166, 0.12562134861946106, 0.22796857357025146, 0.051236074417829514, 0.022322192788124084, 0.08145623654127121, 0.024263404309749603, 0.00022821231686975807, 0.10714813321828842, 0.04663172364234924, 0.005505691282451153, 0.07477705925703049, 0.1910715401172638, 0.15616773068904877, 0.1230965107679367, 0.286618709564209, 0.14264000952243805, 0.1529824584722519, 0.23666103184223175, 0.33499470353126526, 0.06775948405265808, 0.6257417798042297]], "device": "mps:0", "dtype": "torch.bfloat16"}]}, "target": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-7.750509047476794, -0.4993739596732543], "logp": -8.249883007150048, "cache_id": 840, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[411.8838195800781, 128.60980224609375, 118.36662292480469, 142.27590942382812, 51.55754852294922, 39.753639221191406, 204.92213439941406, 260.75537109375, 62.839900970458984, 669.7349243164062, 694.6262817382812, 93.17864227294922, 135.43772888183594, 236.87692260742188, 54.92435836791992, 22.82176399230957, 94.26825714111328, 25.40935516357422, 0.4272097945213318, 105.65047454833984, 45.25754165649414, 5.396683216094971, 78.68418884277344, 180.51022338867188, 138.62718200683594, 105.86917114257812, 292.4052734375, 132.739013671875, 173.25091552734375, 249.2261505126953, 316.7022399902344, 70.60674285888672, 597.735595703125]], "actual_norm": [[411.8750915527344, 128.61460876464844, 118.34187316894531, 142.2723388671875, 51.56538391113281, 39.817039489746094, 204.8722686767578, 260.70819091796875, 62.85100555419922, 669.785888671875, 694.6476440429688, 93.1737060546875, 135.44766235351562, 236.87062072753906, 54.96247482299805, 22.90073585510254, 94.2580795288086, 25.52640151977539, 0.24288122355937958, 105.6361312866211, 45.29419708251953, 5.501523017883301, 78.71949768066406, 180.50009155273438, 138.7248077392578, 105.84021759033203, 292.4077453613281, 132.728759765625, 173.36131286621094, 249.2428436279297, 316.71759033203125, 70.60920715332031, 597.7246704101562]], "relative_norm": [[0.02403903752565384, 0.13643164932727814, 0.13092300295829773, 0.12404660135507584, 0.0469386912882328, 0.03459049388766289, 0.18221107125282288, 0.2315887063741684, 0.04824884608387947, 0.7124463319778442, 0.6619107127189636, 0.09382154047489166, 0.12562134861946106, 0.22796857357025146, 0.051236074417829514, 0.022322192788124084, 0.08145623654127121, 0.024263404309749603, 0.00022821231686975807, 0.10714813321828842, 0.04663172364234924, 0.005505691282451153, 0.07477705925703049, 0.1910715401172638, 0.15616773068904877, 0.1230965107679367, 0.286618709564209, 0.14264000952243805, 0.1529824584722519, 0.23666103184223175, 0.33499470353126526, 0.06775948405265808, 0.6257417798042297]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 23.031876040326743, "first_log_odds": 23.53125, "later_log_odds": -0.4993739596732567, "first_differing_log_odds": 23.53125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-14-paraphrase-bridge.npz", "array_sha256": "0f73929d1481832785f50635e38c2428b42d0740f4cbf5074dcfc43feb6194c6", "seconds": 1.1728765410371125, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-baseline", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 841, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.031204639268008684, -1.3325595402406837e-05], "logp": -0.03121796486341109, "cache_id": 842, "trace": []}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.281204639268008, -0.883906748992367], "logp": -20.165111388260375, "cache_id": 843, "trace": []}, "log_odds": -20.133893423396962, "first_log_odds": -19.25, "later_log_odds": -0.8838934233969624, "first_differing_log_odds": -19.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-baseline.npz", "array_sha256": "99c0da50a44d6aaa00f123d5a41a52a233a394b4c1d27bbfc118dd3e62d5e5d2", "seconds": 0.3750839168205857, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457075712, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-noop", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.539079666137695, "match_error": -9.539079666137695, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 844, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.031204639268008684, -1.3325595402406837e-05], "logp": -0.03121796486341109, "cache_id": 845, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.539079666137695, "match_error": -9.539079666137695, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.281204639268008, -0.883906748992367], "logp": -20.165111388260375, "cache_id": 846, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.539079666137695, "match_error": -9.539079666137695, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.133893423396962, "first_log_odds": -19.25, "later_log_odds": -0.8838934233969624, "first_differing_log_odds": -19.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-noop.npz", "array_sha256": "bda58e3e1d9853dbf55b90ed5f9e9fc0c6962d9cb221c84dfe36904e8b6337a5", "seconds": 0.3381894170306623, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457108480, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-j", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.52575969696045, "actual_norm": 9.539079666137695, "relative_norm": 0.0499963831478442, "target_actual_norm": 9.539079666137695, "match_error": 0.0, "scalar": 0.1837445720967622, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 847, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.04157266119257773, -2.4701581433021134e-05], "logp": -0.04159736277401075, "cache_id": 848, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.52575969696045, "actual_norm": 9.539079666137695, "relative_norm": 0.0499963831478442, "target_actual_norm": 9.539079666137695, "match_error": 0.0, "scalar": 0.1837445720967622, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.35407266119258, -0.4823466488751081], "logp": -17.836419310067686, "cache_id": 849, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.52575969696045, "actual_norm": 9.539079666137695, "relative_norm": 0.0499963831478442, "target_actual_norm": 9.539079666137695, "match_error": 0.0, "scalar": 0.1837445720967622, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.794821947293677, "first_log_odds": -17.3125, "later_log_odds": -0.48232194729367706, "first_differing_log_odds": -17.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-j.npz", "array_sha256": "e3c622da37f0327f8f594280539c61a4ca15aa0c7b4bc9d57a0a740aaab18d1d", "seconds": 0.33695945888757706, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457108480, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-opposite", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.516663551330566, "actual_norm": 9.53799819946289, "relative_norm": 0.04999071494670451, "target_actual_norm": 9.539079666137695, "match_error": -0.0010814666748046875, "scalar": 9.516664226248395, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 850, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.024315381024285695, -6.380813701248461e-06], "logp": -0.024321761837986944, "cache_id": 851, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.516663551330566, "actual_norm": 9.53799819946289, "relative_norm": 0.04999071494670451, "target_actual_norm": 9.539079666137695, "match_error": -0.0010814666748046875, "scalar": 9.516664226248395, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-20.399315381024287, -1.167943355958887], "logp": -21.567258736983174, "cache_id": 852, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.516663551330566, "actual_norm": 9.53799819946289, "relative_norm": 0.04999071494670451, "target_actual_norm": 9.539079666137695, "match_error": -0.0010814666748046875, "scalar": 9.516664226248395, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.54293697514519, "first_log_odds": -20.375, "later_log_odds": -1.1679369751451887, "first_differing_log_odds": -20.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-opposite.npz", "array_sha256": "e2dee61962b116d32da618008f617ac7c793dab38d09ace94a46bb004a37507c", "seconds": 0.3366126250475645, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457108480, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-unrelated", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.527361869812012, "actual_norm": 9.539154052734375, "relative_norm": 0.04999677302411307, "target_actual_norm": 9.539079666137695, "match_error": 7.43865966796875e-05, "scalar": 9.527362504377379, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 853, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.03488760036058632, -1.328459025270403e-05], "logp": -0.034900884950839026, "cache_id": 854, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.527361869812012, "actual_norm": 9.539154052734375, "relative_norm": 0.04999677302411307, "target_actual_norm": 9.539079666137695, "match_error": 7.43865966796875e-05, "scalar": 9.527362504377379, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.347387600360587, -0.9064745977667323], "logp": -20.25386219812732, "cache_id": 855, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.527361869812012, "actual_norm": 9.539154052734375, "relative_norm": 0.04999677302411307, "target_actual_norm": 9.539079666137695, "match_error": 7.43865966796875e-05, "scalar": 9.527362504377379, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.21896131317648, "first_log_odds": -19.3125, "later_log_odds": -0.9064613131764787, "first_differing_log_odds": -19.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-unrelated.npz", "array_sha256": "ef320d977de642b305a820a8619bf97b979a25c6b8989d1a2c7dd5b057029d07", "seconds": 0.3388397917151451, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457108480, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-ordinary", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.539952278137207, "actual_norm": 9.53908634185791, "relative_norm": 0.04999641813674013, "target_actual_norm": 9.539079666137695, "match_error": 6.67572021484375e-06, "scalar": 9.539952994964551, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 856, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.02639955762575482, -1.3300588419333566e-05], "logp": -0.026412858214174155, "cache_id": 857, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.539952278137207, "actual_norm": 9.53908634185791, "relative_norm": 0.04999641813674013, "target_actual_norm": 9.539079666137695, "match_error": 6.67572021484375e-06, "scalar": 9.539952994964551, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.088899557625755, -0.7071392416313904], "logp": -19.796038799257147, "cache_id": 858, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.539952278137207, "actual_norm": 9.53908634185791, "relative_norm": 0.04999641813674013, "target_actual_norm": 9.539079666137695, "match_error": 6.67572021484375e-06, "scalar": 9.539952994964551, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.76962594104297, "first_log_odds": -19.0625, "later_log_odds": -0.7071259410429711, "first_differing_log_odds": -19.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-ordinary.npz", "array_sha256": "e20854eee8c7e8d10a8d52d1acfb45af0d9bd738b8799aff19c75535f1433abe", "seconds": 0.3421307080425322, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457141248, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-random0", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.54130744934082, "actual_norm": 9.539047241210938, "relative_norm": 0.049996213201778286, "target_actual_norm": 9.539079666137695, "match_error": -3.24249267578125e-05, "scalar": 9.541308474081234, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 859, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.02680292065739123, -1.5069455430975748e-05], "logp": -0.026817990112822206, "cache_id": 860, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.54130744934082, "actual_norm": 9.539047241210938, "relative_norm": 0.049996213201778286, "target_actual_norm": 9.539079666137695, "match_error": -3.24249267578125e-05, "scalar": 9.541308474081234, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.464302920657392, -0.7177835429781891], "logp": -20.18208646363558, "cache_id": 861, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.54130744934082, "actual_norm": 9.539047241210938, "relative_norm": 0.049996213201778286, "target_actual_norm": 9.539079666137695, "match_error": -3.24249267578125e-05, "scalar": 9.541308474081234, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.15526847352276, "first_log_odds": -19.4375, "later_log_odds": -0.7177684735227601, "first_differing_log_odds": -19.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-random0.npz", "array_sha256": "aa4981b7e45ff90fb970e1fa7dddfc431e1893d3ac6230fc19d08220ef819004", "seconds": 0.342375542037189, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457141248, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-random1", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.544974327087402, "actual_norm": 9.53896427154541, "relative_norm": 0.049995778339786086, "target_actual_norm": 9.539079666137695, "match_error": -0.00011539459228515625, "scalar": 9.544974635718972, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 862, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.022750286500150093, -9.193574723999455e-06], "logp": -0.02275948007487409, "cache_id": 863, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.544974327087402, "actual_norm": 9.53896427154541, "relative_norm": 0.049995778339786086, "target_actual_norm": 9.539079666137695, "match_error": -0.00011539459228515625, "scalar": 9.544974635718972, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-19.39775028650015, -0.8321384593114851], "logp": -20.229888745811632, "cache_id": 864, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.544974327087402, "actual_norm": 9.53896427154541, "relative_norm": 0.049995778339786086, "target_actual_norm": 9.539079666137695, "match_error": -0.00011539459228515625, "scalar": 9.544974635718972, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.207129265736757, "first_log_odds": -19.375, "later_log_odds": -0.8321292657367572, "first_differing_log_odds": -19.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-random1.npz", "array_sha256": "3bd28fd51d5fadd1e88a3ab9970707f7e94ab9433b283d4ef85ae25fd6678a94", "seconds": 0.3394242920912802, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457141248, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-random2", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.534605979919434, "actual_norm": 9.539206504821777, "relative_norm": 0.049997047936866754, "target_actual_norm": 9.539079666137695, "match_error": 0.00012683868408203125, "scalar": 9.534603855900059, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 865, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.03275074758495561, -1.3312704236554348e-05], "logp": -0.03276406028919216, "cache_id": 866, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.534605979919434, "actual_norm": 9.539206504821777, "relative_norm": 0.049997047936866754, "target_actual_norm": 9.539079666137695, "match_error": 0.00012683868408203125, "scalar": 9.534603855900059, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-18.907750747584956, -0.7639988380117112], "logp": -19.671749585596668, "cache_id": 867, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 190.79539489746094, "strength_one_norm": 51.8424186706543, "target_shrink_scale": 0.1837445720967622, "requested_norm": 9.534605979919434, "actual_norm": 9.539206504821777, "relative_norm": 0.049997047936866754, "target_actual_norm": 9.539079666137695, "match_error": 0.00012683868408203125, "scalar": 9.534603855900059, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -19.638985525307476, "first_log_odds": -18.875, "later_log_odds": -0.7639855253074757, "first_differing_log_odds": -18.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-random2.npz", "array_sha256": "fb9b99e96efe3e891ce59053ec1c064c7bc4c054ca6f3b266730780bcbd86dc5", "seconds": 0.3386903749778867, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457141248, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-main-bridge", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Vietnam Banknote (VND)<|im_end|>", "output_text": "Vietnam Banknote (VND)", "output_ids": [94054, 12400, 8547, 9974, 320, 53, 8065, 8, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 857.632080078125, 19.61102867126465, 269.4883117675781, 22.505985260009766, 14.849183082580566, 22.84638786315918, 62.4854736328125, 191.97645568847656, 59.81519317626953, 199.25665283203125, 73.5422592163086, 0.46598806977272034, 127.98668670654297, 67.30915069580078, 110.49476623535156, 182.02999877929688, 139.29701232910156, 150.2022705078125, 341.02227783203125, 49.83450698852539, 170.2784881591797, 289.7878723144531, 30.22909164428711, 195.76197814941406, 146.7284393310547, 1093.081787109375]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 857.6561279296875, 19.638856887817383, 269.46240234375, 22.54392433166504, 14.893109321594238, 22.890979766845703, 62.48350143432617, 191.8621063232422, 59.886009216308594, 199.25514221191406, 73.5204849243164, 0.3009924590587616, 127.98965454101562, 67.34098815917969, 110.45133972167969, 181.99667358398438, 139.3021697998047, 150.2124481201172, 341.00732421875, 49.88392639160156, 170.3133087158203, 289.7496643066406, 30.2178955078125, 195.75120544433594, 146.68553161621094, 1093.1448974609375]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.7858148217201233, 0.02028035745024681, 0.26490309834480286, 0.02140292339026928, 0.014458993449807167, 0.0221383199095726, 0.05498144030570984, 0.1872471421957016, 0.05708548426628113, 0.2023506760597229, 0.07609483599662781, 0.0003014480462297797, 0.12360081821680069, 0.06418954581022263, 0.08943098783493042, 0.1831851750612259, 0.15607449412345886, 0.1660371720790863, 0.4033106565475464, 0.049429088830947876, 0.18098601698875427, 0.2571485638618469, 0.02922442927956581, 0.21013066172599792, 0.14179173111915588, 0.9088777899742126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 868, "cached": true, "scoring": {"normalized": "Vietnam Banknote (VND)", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Vietnam Banknote (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnam", "start": 0, "end": 7}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-34.31422679883338, -7.61280536191871e-05], "logp": -34.314302926887, "cache_id": 869, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 857.632080078125, 19.61102867126465, 269.4883117675781, 22.505985260009766, 14.849183082580566, 22.84638786315918, 62.4854736328125, 191.97645568847656, 59.81519317626953, 199.25665283203125, 73.5422592163086, 0.46598806977272034, 127.98668670654297, 67.30915069580078, 110.49476623535156, 182.02999877929688, 139.29701232910156, 150.2022705078125, 341.02227783203125, 49.83450698852539, 170.2784881591797, 289.7878723144531, 30.22909164428711, 195.76197814941406, 146.7284393310547, 1093.081787109375]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 857.6561279296875, 19.638856887817383, 269.46240234375, 22.54392433166504, 14.893109321594238, 22.890979766845703, 62.48350143432617, 191.8621063232422, 59.886009216308594, 199.25514221191406, 73.5204849243164, 0.3009924590587616, 127.98965454101562, 67.34098815917969, 110.45133972167969, 181.99667358398438, 139.3021697998047, 150.2124481201172, 341.00732421875, 49.88392639160156, 170.3133087158203, 289.7496643066406, 30.2178955078125, 195.75120544433594, 146.68553161621094, 1093.1448974609375]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.7858148217201233, 0.02028035745024681, 0.26490309834480286, 0.02140292339026928, 0.014458993449807167, 0.0221383199095726, 0.05498144030570984, 0.1872471421957016, 0.05708548426628113, 0.2023506760597229, 0.07609483599662781, 0.0003014480462297797, 0.12360081821680069, 0.06418954581022263, 0.08943098783493042, 0.1831851750612259, 0.15607449412345886, 0.1660371720790863, 0.4033106565475464, 0.049429088830947876, 0.18098601698875427, 0.2571485638618469, 0.02922442927956581, 0.21013066172599792, 0.14179173111915588, 0.9088777899742126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-2.2517267988333836, -0.38258143831515473], "logp": -2.6343082371485385, "cache_id": 870, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 857.632080078125, 19.61102867126465, 269.4883117675781, 22.505985260009766, 14.849183082580566, 22.84638786315918, 62.4854736328125, 191.97645568847656, 59.81519317626953, 199.25665283203125, 73.5422592163086, 0.46598806977272034, 127.98668670654297, 67.30915069580078, 110.49476623535156, 182.02999877929688, 139.29701232910156, 150.2022705078125, 341.02227783203125, 49.83450698852539, 170.2784881591797, 289.7878723144531, 30.22909164428711, 195.76197814941406, 146.7284393310547, 1093.081787109375]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 857.6561279296875, 19.638856887817383, 269.46240234375, 22.54392433166504, 14.893109321594238, 22.890979766845703, 62.48350143432617, 191.8621063232422, 59.886009216308594, 199.25514221191406, 73.5204849243164, 0.3009924590587616, 127.98965454101562, 67.34098815917969, 110.45133972167969, 181.99667358398438, 139.3021697998047, 150.2124481201172, 341.00732421875, 49.88392639160156, 170.3133087158203, 289.7496643066406, 30.2178955078125, 195.75120544433594, 146.68553161621094, 1093.1448974609375]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.7858148217201233, 0.02028035745024681, 0.26490309834480286, 0.02140292339026928, 0.014458993449807167, 0.0221383199095726, 0.05498144030570984, 0.1872471421957016, 0.05708548426628113, 0.2023506760597229, 0.07609483599662781, 0.0003014480462297797, 0.12360081821680069, 0.06418954581022263, 0.08943098783493042, 0.1831851750612259, 0.15607449412345886, 0.1660371720790863, 0.4033106565475464, 0.049429088830947876, 0.18098601698875427, 0.2571485638618469, 0.02922442927956581, 0.21013066172599792, 0.14179173111915588, 0.9088777899742126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 31.679994689738464, "first_log_odds": 32.0625, "later_log_odds": -0.38250531026153567, "first_differing_log_odds": 32.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-main-bridge.npz", "array_sha256": "8414b8ce7424c0944793557f2bcbc198f9d5aa54ba79112720291ee007ed999d", "seconds": 0.5478507080115378, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-baseline", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 871, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-11.625026052868806, -2.460772477912178e-05], "logp": -11.625050660593585, "cache_id": 872, "trace": []}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.062526052868808, -0.9738626170131542], "logp": -26.036388669881962, "cache_id": 873, "trace": []}, "log_odds": -14.411338009288377, "first_log_odds": -13.437500000000002, "later_log_odds": -0.9738380092883752, "first_differing_log_odds": -13.437500000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-baseline.npz", "array_sha256": "a063d7bb2b204600ed7884aca1c6827a7c6c0195a0cad1cde590728022694761", "seconds": 1.224774125032127, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-noop", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.529043197631836, "match_error": -9.529043197631836, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 874, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-11.625026052868806, -2.460772477912178e-05], "logp": -11.625050660593585, "cache_id": 875, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.529043197631836, "match_error": -9.529043197631836, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.062526052868808, -0.9738626170131542], "logp": -26.036388669881962, "cache_id": 876, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.529043197631836, "match_error": -9.529043197631836, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.411338009288377, "first_log_odds": -13.437500000000002, "later_log_odds": -0.9738380092883752, "first_differing_log_odds": -13.437500000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-noop.npz", "array_sha256": "159dd0e36a237f7468b82796a730d465a546cbec83c6a756f44be8d27d42cbf7", "seconds": 1.1832484160549939, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-j", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.510883331298828, "actual_norm": 9.529043197631836, "relative_norm": 0.049982826029903955, "target_actual_norm": 9.529043197631836, "match_error": 0.0, "scalar": 0.1966184451861367, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 877, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-12.87501150625196, -4.5885092295365414e-05], "logp": -12.875057391344255, "cache_id": 878, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.510883331298828, "actual_norm": 9.529043197631836, "relative_norm": 0.049982826029903955, "target_actual_norm": 9.529043197631836, "match_error": 0.0, "scalar": 0.1966184451861367, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.00001150625196, -0.5042301911381982], "logp": -25.504241697390157, "cache_id": 879, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.510883331298828, "actual_norm": 9.529043197631836, "relative_norm": 0.049982826029903955, "target_actual_norm": 9.529043197631836, "match_error": 0.0, "scalar": 0.1966184451861367, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.629184306045902, "first_log_odds": -12.124999999999998, "later_log_odds": -0.5041843060459037, "first_differing_log_odds": -12.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-j.npz", "array_sha256": "33a0d37860489d985df577803c3428e578e909afd45d5d1097a429dabe1964e4", "seconds": 1.1760447919368744, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-opposite", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.51290225982666, "actual_norm": 9.529166221618652, "relative_norm": 0.049983471329374536, "target_actual_norm": 9.529043197631836, "match_error": 0.00012302398681640625, "scalar": 9.512903613358503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 880, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-11.000037020511988, -2.17631857093977e-05], "logp": -11.000058783697696, "cache_id": 881, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.51290225982666, "actual_norm": 9.529166221618652, "relative_norm": 0.049983471329374536, "target_actual_norm": 9.529043197631836, "match_error": 0.00012302398681640625, "scalar": 9.512903613358503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.62503702051199, -1.3751367422335496], "logp": -27.00017376274554, "cache_id": 882, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.51290225982666, "actual_norm": 9.529166221618652, "relative_norm": 0.049983471329374536, "target_actual_norm": 9.529043197631836, "match_error": 0.00012302398681640625, "scalar": 9.512903613358503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.000114979047844, "first_log_odds": -14.625000000000002, "later_log_odds": -1.3751149790478419, "first_differing_log_odds": -14.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-opposite.npz", "array_sha256": "3e7f5da5b62dd1f187002bfb4d413e4d6edb64ddf3b8845d7874191e7d476a46", "seconds": 1.1929566669277847, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-unrelated", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.517050743103027, "actual_norm": 9.529241561889648, "relative_norm": 0.049983866512771244, "target_actual_norm": 9.529043197631836, "match_error": 0.0001983642578125, "scalar": 9.517047560671926, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 883, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-12.250015720136728, -2.787506488631952e-05], "logp": -12.250043595201614, "cache_id": 884, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.517050743103027, "actual_norm": 9.529241561889648, "relative_norm": 0.049983866512771244, "target_actual_norm": 9.529043197631836, "match_error": 0.0001983642578125, "scalar": 9.517047560671926, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.437515720136727, -1.0424481321250167], "logp": -26.479963852261744, "cache_id": 885, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.517050743103027, "actual_norm": 9.529241561889648, "relative_norm": 0.049983866512771244, "target_actual_norm": 9.529043197631836, "match_error": 0.0001983642578125, "scalar": 9.517047560671926, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.22992025706013, "first_log_odds": -13.187499999999998, "later_log_odds": -1.0424202570601313, "first_differing_log_odds": -13.187499999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-unrelated.npz", "array_sha256": "1ae3ff371de3ca862dbbe51e639f9a7f486ff2db0701ede516e7a96128040651", "seconds": 1.1542402496561408, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-ordinary", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.525699615478516, "actual_norm": 9.529038429260254, "relative_norm": 0.04998280101829657, "target_actual_norm": 9.529043197631836, "match_error": -4.76837158203125e-06, "scalar": 9.525698959449073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 886, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-11.8750220068661, -3.151899769010361e-05], "logp": -11.875053525863791, "cache_id": 887, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.525699615478516, "actual_norm": 9.529038429260254, "relative_norm": 0.04998280101829657, "target_actual_norm": 9.529043197631836, "match_error": -4.76837158203125e-06, "scalar": 9.525698959449073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.0000220068661, -0.9005169713798304], "logp": -25.90053897824593, "cache_id": 888, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.525699615478516, "actual_norm": 9.529038429260254, "relative_norm": 0.04998280101829657, "target_actual_norm": 9.529043197631836, "match_error": -4.76837158203125e-06, "scalar": 9.525698959449073, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.025485452382139, "first_log_odds": -13.125, "later_log_odds": -0.9004854523821386, "first_differing_log_odds": -13.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-ordinary.npz", "array_sha256": "09df7c7a43d1e9cda631776fe660562e5aa8c32a63d0abdf3e1a6152bbdab27f", "seconds": 1.1888155830092728, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-random0", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.529845237731934, "actual_norm": 9.528948783874512, "relative_norm": 0.0499823308000777, "target_actual_norm": 9.529043197631836, "match_error": -9.441375732421875e-05, "scalar": 9.529842906762497, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 889, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-12.250014160245213, -2.7879760427269515e-05], "logp": -12.25004204000564, "cache_id": 890, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.529845237731934, "actual_norm": 9.528948783874512, "relative_norm": 0.0499823308000777, "target_actual_norm": 9.529043197631836, "match_error": -9.441375732421875e-05, "scalar": 9.529842906762497, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.750014160245215, -1.1199115388177279], "logp": -26.869925699062943, "cache_id": 891, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.529845237731934, "actual_norm": 9.528948783874512, "relative_norm": 0.0499823308000777, "target_actual_norm": 9.529043197631836, "match_error": -9.441375732421875e-05, "scalar": 9.529842906762497, "random_seed": 20272408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.619883659057303, "first_log_odds": -13.500000000000002, "later_log_odds": -1.119883659057301, "first_differing_log_odds": -13.500000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-random0.npz", "array_sha256": "e31781ea18160ea2b814ab737c4e3ae7ebb1b61242ef0332450c485701951adc", "seconds": 1.1962204999290407, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-random1", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.537513732910156, "actual_norm": 9.529060363769531, "relative_norm": 0.04998291607169055, "target_actual_norm": 9.529043197631836, "match_error": 1.71661376953125e-05, "scalar": 9.537512844333833, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 892, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-12.000018400131099, -1.919161083085019e-05], "logp": -12.000037591741929, "cache_id": 893, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.537513732910156, "actual_norm": 9.529060363769531, "relative_norm": 0.04998291607169055, "target_actual_norm": 9.529043197631836, "match_error": 1.71661376953125e-05, "scalar": 9.537512844333833, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.4375184001311, -0.9144308572233439], "logp": -26.351949257354445, "cache_id": 894, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.537513732910156, "actual_norm": 9.529060363769531, "relative_norm": 0.04998291607169055, "target_actual_norm": 9.529043197631836, "match_error": 1.71661376953125e-05, "scalar": 9.537512844333833, "random_seed": 20272409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.351911665612516, "first_log_odds": -13.437500000000002, "later_log_odds": -0.9144116656125139, "first_differing_log_odds": -13.437500000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-random1.npz", "array_sha256": "01b3d75468727d11366d648f18c7abda315b19a591b23e48dd85b83226e34425", "seconds": 1.1900497497990727, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-random2", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.522499084472656, "actual_norm": 9.529053688049316, "relative_norm": 0.04998288105544021, "target_actual_norm": 9.529043197631836, "match_error": 1.049041748046875e-05, "scalar": 9.52250012292643, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 895, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-11.875021813497835, -2.1718741773015114e-05], "logp": -11.875043532239607, "cache_id": 896, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.522499084472656, "actual_norm": 9.529053688049316, "relative_norm": 0.04998288105544021, "target_actual_norm": 9.529043197631836, "match_error": 1.049041748046875e-05, "scalar": 9.52250012292643, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-25.250021813497835, -0.9657336131942383], "logp": -26.215755426692073, "cache_id": 897, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 190.64634704589844, "strength_one_norm": 48.372283935546875, "target_shrink_scale": 0.1966184451861367, "requested_norm": 9.522499084472656, "actual_norm": 9.529053688049316, "relative_norm": 0.04998288105544021, "target_actual_norm": 9.529043197631836, "match_error": 1.049041748046875e-05, "scalar": 9.52250012292643, "random_seed": 20272410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.340711894452467, "first_log_odds": -13.375, "later_log_odds": -0.9657118944524665, "first_differing_log_odds": -13.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-random2.npz", "array_sha256": "8d924c139fd73d99c1eb4b870300034f0f76e239d9cf462e5bd6d7690475ad26", "seconds": 1.191989874932915, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-15-paraphrase-bridge", "case_id": "eval-15", "city": "Bangkok", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Bangkok is the **Bhat** (BDT), and its three-letter ISO 4217 code is **BDT**.<|im_end|>", "output_text": "The currency used in Bangkok is the **Bhat** (BDT), and its three-letter ISO 4217 code is **BDT**.", "output_ids": [785, 11413, 1483, 304, 59016, 374, 279, 3070, 33, 9198, 334, 320, 9548, 51, 701, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 9548, 51, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 828.2591552734375, 53.895198822021484, 25.418607711791992, 34.099857330322266, 5.82742166519165, 18.074546813964844, 32.94050598144531, 149.53062438964844, 40.627723693847656, 179.49343872070312, 74.55897521972656, 6.673555850982666, 178.2763671875, 62.73396301269531, 154.97286987304688, 298.30999755859375, 61.6347770690918, 177.01217651367188, 273.4744567871094, 24.16767120361328, 154.0798797607422, 147.03292846679688, 638.8900756835938]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 828.2341918945312, 53.9080810546875, 25.469314575195312, 34.13264846801758, 5.961496829986572, 18.156494140625, 32.96844482421875, 149.57696533203125, 40.6530647277832, 179.51063537597656, 74.63300323486328, 6.82802152633667, 178.32742309570312, 62.74738693237305, 154.98829650878906, 298.328369140625, 61.58622360229492, 176.9761199951172, 273.5133056640625, 24.268468856811523, 154.10020446777344, 147.01353454589844, 638.8633422851562]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.7536045908927917, 0.05168483778834343, 0.023257916793227196, 0.031977228820323944, 0.005473837722092867, 0.017657088115811348, 0.028347807005047798, 0.14152386784553528, 0.03842666372656822, 0.1811484545469284, 0.07651358097791672, 0.006741124205291271, 0.16935594379901886, 0.06506750732660294, 0.17479479312896729, 0.3440050184726715, 0.06003710255026817, 0.18895801901817322, 0.2404831051826477, 0.023050300776958466, 0.16288341581821442, 0.1411069929599762, 0.6303600072860718]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 898, "cached": true, "scoring": {"normalized": "The currency used in Bangkok is the **Bhat** (BDT), and its three-letter ISO 4217 code is **BDT**.", "bare_format": false, "malformed": true, "first_code": "BDT", "all_codes": ["BDT", "BDT"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Bangkok is the **Bhat** (", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-37.03237922509023, -8.570975069912358e-05], "logp": -37.03246493484093, "cache_id": 899, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 828.2591552734375, 53.895198822021484, 25.418607711791992, 34.099857330322266, 5.82742166519165, 18.074546813964844, 32.94050598144531, 149.53062438964844, 40.627723693847656, 179.49343872070312, 74.55897521972656, 6.673555850982666, 178.2763671875, 62.73396301269531, 154.97286987304688, 298.30999755859375, 61.6347770690918, 177.01217651367188, 273.4744567871094, 24.16767120361328, 154.0798797607422, 147.03292846679688, 638.8900756835938]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 828.2341918945312, 53.9080810546875, 25.469314575195312, 34.13264846801758, 5.961496829986572, 18.156494140625, 32.96844482421875, 149.57696533203125, 40.6530647277832, 179.51063537597656, 74.63300323486328, 6.82802152633667, 178.32742309570312, 62.74738693237305, 154.98829650878906, 298.328369140625, 61.58622360229492, 176.9761199951172, 273.5133056640625, 24.268468856811523, 154.10020446777344, 147.01353454589844, 638.8633422851562]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.7536045908927917, 0.05168483778834343, 0.023257916793227196, 0.031977228820323944, 0.005473837722092867, 0.017657088115811348, 0.028347807005047798, 0.14152386784553528, 0.03842666372656822, 0.1811484545469284, 0.07651358097791672, 0.006741124205291271, 0.16935594379901886, 0.06506750732660294, 0.17479479312896729, 0.3440050184726715, 0.06003710255026817, 0.18895801901817322, 0.2404831051826477, 0.023050300776958466, 0.16288341581821442, 0.1411069929599762, 0.6303600072860718]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-8.126129225090235, -0.2988330850194881], "logp": -8.424962310109724, "cache_id": 900, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 828.2591552734375, 53.895198822021484, 25.418607711791992, 34.099857330322266, 5.82742166519165, 18.074546813964844, 32.94050598144531, 149.53062438964844, 40.627723693847656, 179.49343872070312, 74.55897521972656, 6.673555850982666, 178.2763671875, 62.73396301269531, 154.97286987304688, 298.30999755859375, 61.6347770690918, 177.01217651367188, 273.4744567871094, 24.16767120361328, 154.0798797607422, 147.03292846679688, 638.8900756835938]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 828.2341918945312, 53.9080810546875, 25.469314575195312, 34.13264846801758, 5.961496829986572, 18.156494140625, 32.96844482421875, 149.57696533203125, 40.6530647277832, 179.51063537597656, 74.63300323486328, 6.82802152633667, 178.32742309570312, 62.74738693237305, 154.98829650878906, 298.328369140625, 61.58622360229492, 176.9761199951172, 273.5133056640625, 24.268468856811523, 154.10020446777344, 147.01353454589844, 638.8633422851562]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.7536045908927917, 0.05168483778834343, 0.023257916793227196, 0.031977228820323944, 0.005473837722092867, 0.017657088115811348, 0.028347807005047798, 0.14152386784553528, 0.03842666372656822, 0.1811484545469284, 0.07651358097791672, 0.006741124205291271, 0.16935594379901886, 0.06506750732660294, 0.17479479312896729, 0.3440050184726715, 0.06003710255026817, 0.18895801901817322, 0.2404831051826477, 0.023050300776958466, 0.16288341581821442, 0.1411069929599762, 0.6303600072860718]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 28.607502624731207, "first_log_odds": 28.906249999999996, "later_log_odds": -0.29874737526878903, "first_differing_log_odds": 28.906249999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-15-paraphrase-bridge.npz", "array_sha256": "5b4e536ceb2556f1ea9f05fb45a76357e3476462430804b5f88d8cbae0df1282", "seconds": 1.3038039579987526, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-baseline", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 901, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.01882972161022354, -0.00010786626562809561], "logp": -0.018937587875851633, "cache_id": 902, "trace": []}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.518829721610224, -0.9720389197605304], "logp": -18.490868641370753, "cache_id": 903, "trace": []}, "log_odds": -18.471931053494902, "first_log_odds": -17.5, "later_log_odds": -0.9719310534949024, "first_differing_log_odds": -17.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-baseline.npz", "array_sha256": "d0e9b08a36374cd26f73ccb28e25a24128f99cc0ab44344cbd2ccbc4f1574843", "seconds": 0.3877590000629425, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-noop", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.889249801635742, "match_error": -9.889249801635742, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 904, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.01882972161022354, -0.00010786626562809561], "logp": -0.018937587875851633, "cache_id": 905, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.889249801635742, "match_error": -9.889249801635742, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.518829721610224, -0.9720389197605304], "logp": -18.490868641370753, "cache_id": 906, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.889249801635742, "match_error": -9.889249801635742, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.471931053494902, "first_log_odds": -17.5, "later_log_odds": -0.9719310534949024, "first_differing_log_odds": -17.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-noop.npz", "array_sha256": "d878645ae6ed9faa894f834691c6a72775d5d571eb7fd17e5549511cb04dd41f", "seconds": 0.34156279219314456, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-j", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.873189926147461, "actual_norm": 9.889249801635742, "relative_norm": 0.04999892187484691, "target_actual_norm": 9.889249801635742, "match_error": 0.0, "scalar": 0.1958341225935175, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 907, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.041047912576729996, -0.00022312147378477807], "logp": -0.04127103405051477, "cache_id": 908, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.873189926147461, "actual_norm": 9.889249801635742, "relative_norm": 0.04999892187484691, "target_actual_norm": 9.889249801635742, "match_error": 0.0, "scalar": 0.1958341225935175, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-16.66604791257673, -1.1135718480400039], "logp": -17.779619760616733, "cache_id": 909, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.873189926147461, "actual_norm": 9.889249801635742, "relative_norm": 0.04999892187484691, "target_actual_norm": 9.889249801635742, "match_error": 0.0, "scalar": 0.1958341225935175, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.73834872656622, "first_log_odds": -16.625, "later_log_odds": -1.113348726566219, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-j.npz", "array_sha256": "50092a497f33453847a1bba519737c708d2349df29d3b5082d70138d3cccc4ee", "seconds": 0.33956870902329683, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-opposite", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.869632720947266, "actual_norm": 9.888587951660156, "relative_norm": 0.049995575636670515, "target_actual_norm": 9.889249801635742, "match_error": -0.0006618499755859375, "scalar": 9.869633064663503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 910, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.013352272667670292, -8.444349154495902e-05], "logp": -0.013436716159215252, "cache_id": 911, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.869632720947266, "actual_norm": 9.888587951660156, "relative_norm": 0.049995575636670515, "target_actual_norm": 9.889249801635742, "match_error": -0.0006618499755859375, "scalar": 9.869633064663503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-18.07585227266767, -0.7243932014751984], "logp": -18.80024547414287, "cache_id": 912, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.869632720947266, "actual_norm": 9.888587951660156, "relative_norm": 0.049995575636670515, "target_actual_norm": 9.889249801635742, "match_error": -0.0006618499755859375, "scalar": 9.869633064663503, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.786808757983653, "first_log_odds": -18.0625, "later_log_odds": -0.7243087579836534, "first_differing_log_odds": -18.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-opposite.npz", "array_sha256": "5054f8d67e44661718ba4bbe6ff2bb9ea916c6834eac354179ff04b4eeda757c", "seconds": 0.3447314170189202, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-unrelated", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.894227027893066, "actual_norm": 9.889056205749512, "relative_norm": 0.04999794307607197, "target_actual_norm": 9.889249801635742, "match_error": -0.00019359588623046875, "scalar": 9.894229434867157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 913, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.010703557669692405, -0.00010523967312085311], "logp": -0.010808797342813257, "cache_id": 914, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.894227027893066, "actual_norm": 9.889056205749512, "relative_norm": 0.04999794307607197, "target_actual_norm": 9.889249801635742, "match_error": -0.00019359588623046875, "scalar": 9.894229434867157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-18.010703557669693, -0.8624156636153693], "logp": -18.873119221285062, "cache_id": 915, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.894227027893066, "actual_norm": 9.889056205749512, "relative_norm": 0.04999794307607197, "target_actual_norm": 9.889249801635742, "match_error": -0.00019359588623046875, "scalar": 9.894229434867157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.862310423942247, "first_log_odds": -18.0, "later_log_odds": -0.8623104239422474, "first_differing_log_odds": -18.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-unrelated.npz", "array_sha256": "41e49d37a01513484e781c1bb33fe7f0e45c1c5d3e9df0e0dc82684b4cb05353", "seconds": 0.3407435002736747, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-ordinary", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.886987686157227, "actual_norm": 9.88886833190918, "relative_norm": 0.04999699320731008, "target_actual_norm": 9.889249801635742, "match_error": -0.0003814697265625, "scalar": 9.8869863319851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 916, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.02420142030736936, -0.00012532012766451546], "logp": -0.024326740435033876, "cache_id": 917, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.886987686157227, "actual_norm": 9.88886833190918, "relative_norm": 0.04999699320731008, "target_actual_norm": 9.889249801635742, "match_error": -0.0003814697265625, "scalar": 9.8869863319851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.27420142030737, -0.9012200518022747], "logp": -18.175421472109644, "cache_id": 918, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.886987686157227, "actual_norm": 9.88886833190918, "relative_norm": 0.04999699320731008, "target_actual_norm": 9.889249801635742, "match_error": -0.0003814697265625, "scalar": 9.8869863319851, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.15109473167461, "first_log_odds": -17.25, "later_log_odds": -0.9010947316746112, "first_differing_log_odds": -17.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-ordinary.npz", "array_sha256": "aee6675b6be83b58ef6e7fe45a82eee8755fbdd8591d1857606759e620ff205c", "seconds": 0.34158383402973413, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-random0", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.89184856414795, "actual_norm": 9.889264106750488, "relative_norm": 0.04999899419987954, "target_actual_norm": 9.889249801635742, "match_error": 1.430511474609375e-05, "scalar": 9.89184807617221, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 919, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.025719472023166084, -9.480356497201541e-05], "logp": -0.0258142755881381, "cache_id": 920, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.89184856414795, "actual_norm": 9.889264106750488, "relative_norm": 0.04999899419987954, "target_actual_norm": 9.889249801635742, "match_error": 1.430511474609375e-05, "scalar": 9.89184807617221, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.150719472023166, -0.859343458037363], "logp": -18.010062930060528, "cache_id": 921, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.89184856414795, "actual_norm": 9.889264106750488, "relative_norm": 0.04999899419987954, "target_actual_norm": 9.889249801635742, "match_error": 1.430511474609375e-05, "scalar": 9.89184807617221, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.98424865447239, "first_log_odds": -17.125, "later_log_odds": -0.8592486544723883, "first_differing_log_odds": -17.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-random0.npz", "array_sha256": "7af194b207d55d944a089f316df771d98025c939e6f9e96619c9aff44e3cabc5", "seconds": 0.34227799996733665, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-random1", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.893890380859375, "actual_norm": 9.889093399047852, "relative_norm": 0.049998131121156815, "target_actual_norm": 9.889249801635742, "match_error": -0.000156402587890625, "scalar": 9.89388991441956, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 922, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.012772670027647978, -0.00014153034070933987], "logp": -0.012914200368357318, "cache_id": 923, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.893890380859375, "actual_norm": 9.889093399047852, "relative_norm": 0.049998131121156815, "target_actual_norm": 9.889249801635742, "match_error": -0.000156402587890625, "scalar": 9.89388991441956, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.950272670027648, -0.7969618745341298], "logp": -18.747234544561778, "cache_id": 924, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.893890380859375, "actual_norm": 9.889093399047852, "relative_norm": 0.049998131121156815, "target_actual_norm": 9.889249801635742, "match_error": -0.000156402587890625, "scalar": 9.89388991441956, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.73432034419342, "first_log_odds": -17.9375, "later_log_odds": -0.7968203441934207, "first_differing_log_odds": -17.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-random1.npz", "array_sha256": "1ce2a5bde595e0b07982337ed6105146cb3fecf283e67a0fa5780f3886218741", "seconds": 0.3393438751809299, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-random2", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "THB<|im_end|>", "output_text": "THB", "output_ids": [3617, 33, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.87699031829834, "actual_norm": 9.889244079589844, "relative_norm": 0.04999889294483386, "target_actual_norm": 9.889249801635742, "match_error": -5.7220458984375e-06, "scalar": 9.876989341028093, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 925, "cached": true, "scoring": {"normalized": "THB", "bare_format": true, "malformed": false, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-0.013364057614441432, -0.00014157244662763954], "logp": -0.013505630061069071, "cache_id": 926, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.87699031829834, "actual_norm": 9.889244079589844, "relative_norm": 0.04999889294483386, "target_actual_norm": 9.889249801635742, "match_error": -5.7220458984375e-06, "scalar": 9.876989341028093, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.888364057614442, -0.949160543079571], "logp": -18.837524600694014, "cache_id": 927, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 197.7892608642578, "strength_one_norm": 50.41607666015625, "target_shrink_scale": 0.1958341225935175, "requested_norm": 9.87699031829834, "actual_norm": 9.889244079589844, "relative_norm": 0.04999889294483386, "target_actual_norm": 9.889249801635742, "match_error": -5.7220458984375e-06, "scalar": 9.876989341028093, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.824018970632945, "first_log_odds": -17.875, "later_log_odds": -0.9490189706329453, "first_differing_log_odds": -17.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-random2.npz", "array_sha256": "6b8f296649a62326bf1fba734d5bf6de181779ff21ebb47238958f7ac0a2e537", "seconds": 0.33966641686856747, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-main-bridge", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 450.7174072265625, 963.1397705078125, 51.3341064453125, 258.0928955078125, 32.415348052978516, 66.52557373046875, 10.276372909545898, 73.82704162597656, 204.33238220214844, 73.44941711425781, 204.23333740234375, 64.47655487060547, 3.5189030170440674, 131.0224609375, 46.971397399902344, 97.07041931152344, 172.48768615722656, 131.73001098632812, 132.17617797851562, 355.99822998046875, 48.421566009521484, 164.5498046875, 289.3025207519531, 23.232524871826172, 229.99160766601562, 143.02166748046875, 1020.1483764648438]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 450.7729797363281, 963.1791381835938, 51.342472076416016, 258.0707702636719, 32.465606689453125, 66.5394287109375, 10.417146682739258, 73.81288146972656, 204.34959411621094, 73.4241714477539, 204.25820922851562, 64.5046615600586, 3.707507371902466, 131.0142822265625, 46.95637130737305, 97.13371276855469, 172.507568359375, 131.73228454589844, 132.19415283203125, 355.9581604003906, 48.39993667602539, 164.52772521972656, 289.2613830566406, 23.265806198120117, 229.93472290039062, 143.01504516601562, 1020.0722045898438]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.3293640911579132, 0.8897833228111267, 0.053503844887018204, 0.24850423634052277, 0.030736278742551804, 0.0645509585738182, 0.01016195211559534, 0.0651434138417244, 0.2029852569103241, 0.07031432539224625, 0.20802463591098785, 0.06727024912834167, 0.0037348493933677673, 0.12651056051254272, 0.04467374086380005, 0.0783408060669899, 0.17304742336273193, 0.1466478556394577, 0.14741578698158264, 0.4189126789569855, 0.04820805415511131, 0.17403925955295563, 0.25502869486808777, 0.022656748071312904, 0.24822817742824554, 0.13807781040668488, 0.9420754313468933]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 928, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-36.294698469347374, -0.0003607401879198048], "logp": -36.295059209535296, "cache_id": 929, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 450.7174072265625, 963.1397705078125, 51.3341064453125, 258.0928955078125, 32.415348052978516, 66.52557373046875, 10.276372909545898, 73.82704162597656, 204.33238220214844, 73.44941711425781, 204.23333740234375, 64.47655487060547, 3.5189030170440674, 131.0224609375, 46.971397399902344, 97.07041931152344, 172.48768615722656, 131.73001098632812, 132.17617797851562, 355.99822998046875, 48.421566009521484, 164.5498046875, 289.3025207519531, 23.232524871826172, 229.99160766601562, 143.02166748046875, 1020.1483764648438]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 450.7729797363281, 963.1791381835938, 51.342472076416016, 258.0707702636719, 32.465606689453125, 66.5394287109375, 10.417146682739258, 73.81288146972656, 204.34959411621094, 73.4241714477539, 204.25820922851562, 64.5046615600586, 3.707507371902466, 131.0142822265625, 46.95637130737305, 97.13371276855469, 172.507568359375, 131.73228454589844, 132.19415283203125, 355.9581604003906, 48.39993667602539, 164.52772521972656, 289.2613830566406, 23.265806198120117, 229.93472290039062, 143.01504516601562, 1020.0722045898438]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.3293640911579132, 0.8897833228111267, 0.053503844887018204, 0.24850423634052277, 0.030736278742551804, 0.0645509585738182, 0.01016195211559534, 0.0651434138417244, 0.2029852569103241, 0.07031432539224625, 0.20802463591098785, 0.06727024912834167, 0.0037348493933677673, 0.12651056051254272, 0.04467374086380005, 0.0783408060669899, 0.17304742336273193, 0.1466478556394577, 0.14741578698158264, 0.4189126789569855, 0.04820805415511131, 0.17403925955295563, 0.25502869486808777, 0.022656748071312904, 0.24822817742824554, 0.13807781040668488, 0.9420754313468933]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.31813596934737454, -0.30809388208749194], "logp": -0.6262298514348665, "cache_id": 930, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 38.361663818359375, 4.689302444458008, 50.68971633911133, 36.95232009887695, 87.7063217163086, 450.7174072265625, 963.1397705078125, 51.3341064453125, 258.0928955078125, 32.415348052978516, 66.52557373046875, 10.276372909545898, 73.82704162597656, 204.33238220214844, 73.44941711425781, 204.23333740234375, 64.47655487060547, 3.5189030170440674, 131.0224609375, 46.971397399902344, 97.07041931152344, 172.48768615722656, 131.73001098632812, 132.17617797851562, 355.99822998046875, 48.421566009521484, 164.5498046875, 289.3025207519531, 23.232524871826172, 229.99160766601562, 143.02166748046875, 1020.1483764648438]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 38.34359359741211, 4.818782806396484, 50.7043342590332, 36.99945068359375, 87.6897964477539, 450.7729797363281, 963.1791381835938, 51.342472076416016, 258.0707702636719, 32.465606689453125, 66.5394287109375, 10.417146682739258, 73.81288146972656, 204.34959411621094, 73.4241714477539, 204.25820922851562, 64.5046615600586, 3.707507371902466, 131.0142822265625, 46.95637130737305, 97.13371276855469, 172.507568359375, 131.73228454589844, 132.19415283203125, 355.9581604003906, 48.39993667602539, 164.52772521972656, 289.2613830566406, 23.265806198120117, 229.93472290039062, 143.01504516601562, 1020.0722045898438]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03519807010889053, 0.004227910190820694, 0.04125066474080086, 0.03166858106851578, 0.07727231830358505, 0.3293640911579132, 0.8897833228111267, 0.053503844887018204, 0.24850423634052277, 0.030736278742551804, 0.0645509585738182, 0.01016195211559534, 0.0651434138417244, 0.2029852569103241, 0.07031432539224625, 0.20802463591098785, 0.06727024912834167, 0.0037348493933677673, 0.12651056051254272, 0.04467374086380005, 0.0783408060669899, 0.17304742336273193, 0.1466478556394577, 0.14741578698158264, 0.4189126789569855, 0.04820805415511131, 0.17403925955295563, 0.25502869486808777, 0.022656748071312904, 0.24822817742824554, 0.13807781040668488, 0.9420754313468933]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 35.66882935810043, "first_log_odds": 35.9765625, "later_log_odds": -0.30773314189956835, "first_differing_log_odds": 35.9765625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-main-bridge.npz", "array_sha256": "a1ea5947502ac1c43b4ba6ec6f79814ab6e39f0ed7bcab0c1a1885c57ac47297", "seconds": 0.3426629169844091, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-baseline", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 931, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-20.500000036561502, -0.000658781957732248], "logp": -20.500658818519234, "cache_id": 932, "trace": []}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.500000036561502, -0.9371541892134522], "logp": -29.437154225774954, "cache_id": 933, "trace": []}, "log_odds": -8.93649540725572, "first_log_odds": -8.0, "later_log_odds": -0.9364954072557197, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-baseline.npz", "array_sha256": "6884b5fdf53dbdd7cf9586a0e8ac4054374e2ce49bc1e1a34e8f2e262a08cbac", "seconds": 1.227254290599376, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-noop", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.005891799926758, "match_error": -10.005891799926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 934, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-20.500000036561502, -0.000658781957732248], "logp": -20.500658818519234, "cache_id": 935, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.005891799926758, "match_error": -10.005891799926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.500000036561502, -0.9371541892134522], "logp": -29.437154225774954, "cache_id": 936, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.005891799926758, "match_error": -10.005891799926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.93649540725572, "first_log_odds": -8.0, "later_log_odds": -0.9364954072557197, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-noop.npz", "array_sha256": "93a63d4439fb93dcef01ddf0428a501013e306a3f9b82bcd3c5313cb54996885", "seconds": 1.2238816251046956, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-j", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.000802993774414, "actual_norm": 10.005891799926758, "relative_norm": 0.04999997140663351, "target_actual_norm": 10.005891799926758, "match_error": 0.0, "scalar": 0.19084220376384883, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 937, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-21.25000002984809, -0.0015780834533864235], "logp": -21.251578113301473, "cache_id": 938, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.000802993774414, "actual_norm": 10.005891799926758, "relative_norm": 0.04999997140663351, "target_actual_norm": 10.005891799926758, "match_error": 0.0, "scalar": 0.19084220376384883, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.00000002984809, -0.737803984974983], "logp": -28.737804014823073, "cache_id": 939, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.000802993774414, "actual_norm": 10.005891799926758, "relative_norm": 0.04999997140663351, "target_actual_norm": 10.005891799926758, "match_error": 0.0, "scalar": 0.19084220376384883, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.4862259015216, "first_log_odds": -6.75, "later_log_odds": -0.7362259015215997, "first_differing_log_odds": -6.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-j.npz", "array_sha256": "1e52c99b98d006110e04513f7f334847495ae861434dec50e72614601c6a2eef", "seconds": 1.231811833102256, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-opposite", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005815505981445, "actual_norm": 10.003828048706055, "relative_norm": 0.04998965873245225, "target_actual_norm": 10.005891799926758, "match_error": -0.002063751220703125, "scalar": 10.00581546103058, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 940, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-18.500000131862492, -0.00037760033370649204], "logp": -18.5003777321962, "cache_id": 941, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005815505981445, "actual_norm": 10.003828048706055, "relative_norm": 0.04998965873245225, "target_actual_norm": 10.005891799926758, "match_error": -0.002063751220703125, "scalar": 10.00581546103058, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.250000131862492, -0.8641269740934515], "logp": -29.114127105955944, "cache_id": 942, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005815505981445, "actual_norm": 10.003828048706055, "relative_norm": 0.04998965873245225, "target_actual_norm": 10.005891799926758, "match_error": -0.002063751220703125, "scalar": 10.00581546103058, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.613749373759745, "first_log_odds": -9.75, "later_log_odds": -0.8637493737597453, "first_differing_log_odds": -9.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-opposite.npz", "array_sha256": "333b18a827f3b4bc65ae536980f92b35f90dedc3883eb215ebda1bc5caed9b5c", "seconds": 1.2073310832493007, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-unrelated", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.013219833374023, "actual_norm": 10.004358291625977, "relative_norm": 0.04999230838441379, "target_actual_norm": 10.005891799926758, "match_error": -0.00153350830078125, "scalar": 10.013220333959907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 943, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-19.875000058946075, -0.0006615104926573464], "logp": -19.875661569438734, "cache_id": 944, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.013219833374023, "actual_norm": 10.004358291625977, "relative_norm": 0.04999230838441379, "target_actual_norm": 10.005891799926758, "match_error": -0.00153350830078125, "scalar": 10.013220333959907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.250000058946075, -0.862256764952339], "logp": -29.112256823898413, "cache_id": 945, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.013219833374023, "actual_norm": 10.004358291625977, "relative_norm": 0.04999230838441379, "target_actual_norm": 10.005891799926758, "match_error": -0.00153350830078125, "scalar": 10.013220333959907, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.23659525445968, "first_log_odds": -8.375, "later_log_odds": -0.8615952544596794, "first_differing_log_odds": -8.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-unrelated.npz", "array_sha256": "c9e5b363742668d8bf0cb7006b31ae3990aaf1fbd0f6e0cb84efa75936207444", "seconds": 1.240599500015378, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-ordinary", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.993152618408203, "actual_norm": 10.005829811096191, "relative_norm": 0.04999966164516319, "target_actual_norm": 10.005891799926758, "match_error": -6.198883056640625e-05, "scalar": 9.993152746626947, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 946, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-21.00000003239287, -0.0007492571208539005], "logp": -21.000749289513724, "cache_id": 947, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.993152618408203, "actual_norm": 10.005829811096191, "relative_norm": 0.04999966164516319, "target_actual_norm": 10.005891799926758, "match_error": -6.198883056640625e-05, "scalar": 9.993152746626947, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.37500003239287, -0.7401911600632023], "logp": -29.11519119245607, "cache_id": 948, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.993152618408203, "actual_norm": 10.005829811096191, "relative_norm": 0.04999966164516319, "target_actual_norm": 10.005891799926758, "match_error": -6.198883056640625e-05, "scalar": 9.993152746626947, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.114441902942346, "first_log_odds": -7.375, "later_log_odds": -0.739441902942346, "first_differing_log_odds": -7.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-ordinary.npz", "array_sha256": "928ecfea42eb3c366447de418f0ffc71589df879c84e74d91783e09591d095cd", "seconds": 1.251883374992758, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-random0", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.011492729187012, "actual_norm": 10.00584888458252, "relative_norm": 0.049999756956384825, "target_actual_norm": 10.005891799926758, "match_error": -4.291534423828125e-05, "scalar": 10.011493166433866, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 949, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-20.25000004361969, -0.0007951633370140598], "logp": -20.250795206956703, "cache_id": 950, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.011492729187012, "actual_norm": 10.00584888458252, "relative_norm": 0.049999756956384825, "target_actual_norm": 10.005891799926758, "match_error": -4.291534423828125e-05, "scalar": 10.011493166433866, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.25000004361969, -0.8649654469001833], "logp": -29.11496549051987, "cache_id": 951, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.011492729187012, "actual_norm": 10.00584888458252, "relative_norm": 0.049999756956384825, "target_actual_norm": 10.005891799926758, "match_error": -4.291534423828125e-05, "scalar": 10.011493166433866, "random_seed": 20272508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.864170283563169, "first_log_odds": -8.0, "later_log_odds": -0.864170283563169, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-random0.npz", "array_sha256": "0ddbc13513be84284df15f905c8fb46f144730228d9de909aa6c069175056b31", "seconds": 1.2296566660515964, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-random1", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005282402038574, "actual_norm": 10.005882263183594, "relative_norm": 0.04999992375102269, "target_actual_norm": 10.005891799926758, "match_error": -9.5367431640625e-06, "scalar": 10.005281088757329, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 952, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-20.37500004353917, -0.0007589226875145818], "logp": -20.375758966226684, "cache_id": 953, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005282402038574, "actual_norm": 10.005882263183594, "relative_norm": 0.04999992375102269, "target_actual_norm": 10.005891799926758, "match_error": -9.5367431640625e-06, "scalar": 10.005281088757329, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.62500004353917, -0.9312493567786146], "logp": -29.556249400317785, "cache_id": 954, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 10.005282402038574, "actual_norm": 10.005882263183594, "relative_norm": 0.04999992375102269, "target_actual_norm": 10.005891799926758, "match_error": -9.5367431640625e-06, "scalar": 10.005281088757329, "random_seed": 20272509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.180490434091102, "first_log_odds": -8.25, "later_log_odds": -0.9304904340911015, "first_differing_log_odds": -8.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-random1.npz", "array_sha256": "ebc4054009926c3a4678c88386dd1e6f3ffdcf1be04212d4a6a49d387213218f", "seconds": 1.2512660003267229, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-random2", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.997207641601562, "actual_norm": 10.005888938903809, "relative_norm": 0.04999995710995026, "target_actual_norm": 10.005891799926758, "match_error": -2.86102294921875e-06, "scalar": 9.997208250486437, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 955, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-20.250000049971902, -0.0006353555660437933], "logp": -20.250635405537945, "cache_id": 956, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.997207641601562, "actual_norm": 10.005888938903809, "relative_norm": 0.04999995710995026, "target_actual_norm": 10.005891799926758, "match_error": -2.86102294921875e-06, "scalar": 9.997208250486437, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-28.375000049971902, -0.86658808185986], "logp": -29.24158813183176, "cache_id": 957, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 200.11795043945312, "strength_one_norm": 52.40353012084961, "target_shrink_scale": 0.19084220376384883, "requested_norm": 9.997207641601562, "actual_norm": 10.005888938903809, "relative_norm": 0.04999995710995026, "target_actual_norm": 10.005891799926758, "match_error": -2.86102294921875e-06, "scalar": 9.997208250486437, "random_seed": 20272510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.990952726293816, "first_log_odds": -8.125, "later_log_odds": -0.8659527262938163, "first_differing_log_odds": -8.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-random2.npz", "array_sha256": "6e0fcffb9233d9085db942caeae2c64d014ae28c4b9dd6f11634e16bed3142e7", "seconds": 1.2210908331908286, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-16-paraphrase-bridge", "case_id": "eval-16", "city": "Pattaya", "country": "Thailand", "code": "THB", "target_country": "Vietnam", "target_code": "VND", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.<|im_end|>", "output_text": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "output_ids": [785, 11413, 1483, 304, 44303, 12589, 374, 279, 26392, 14322, 426, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 3617, 33, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 497.8238220214844, 990.9141235351562, 90.72662353515625, 46.94430160522461, 55.662818908691406, 11.270968437194824, 14.169890403747559, 51.431861877441406, 171.14283752441406, 43.8043098449707, 184.9853515625, 66.5218505859375, 6.707688331604004, 160.7194061279297, 20.15272331237793, 133.01991271972656, 331.8583984375, 61.999244689941406, 174.69996643066406, 268.67156982421875, 24.691606521606445, 157.39556884765625, 140.01974487304688, 569.4140625]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 497.8292236328125, 990.96826171875, 90.73218536376953, 46.946998596191406, 55.693321228027344, 11.341758728027344, 14.202651977539062, 51.46010971069336, 171.12388610839844, 43.85261535644531, 185.0232391357422, 66.5111312866211, 6.856252670288086, 160.7225341796875, 20.16823959350586, 133.0384063720703, 331.7846374511719, 61.97875213623047, 174.7203369140625, 268.6419982910156, 24.775175094604492, 157.38113403320312, 140.0228729248047, 569.3577270507812]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.3618178069591522, 0.9178438186645508, 0.09484148025512695, 0.04385346546769142, 0.05322195217013359, 0.01051488146185875, 0.013940233737230301, 0.044421568512916565, 0.16253618896007538, 0.04125693067908287, 0.18745487928390503, 0.06889171153306961, 0.006697432138025761, 0.15474571287631989, 0.021488500759005547, 0.1501874178647995, 0.37843865156173706, 0.060740627348423004, 0.18518975377082825, 0.237309068441391, 0.023639803752303123, 0.1654440313577652, 0.1349678784608841, 0.6009939312934875]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 958, "cached": true, "scoring": {"normalized": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **THB**.", "bare_format": false, "malformed": true, "first_code": "THB", "all_codes": ["THB"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Pattaya is the Thai Baht, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Thai", "start": 36, "end": 40}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [3617, 33], "tokens": ["TH", "B"], "log_probs": [-40.92189319110674, -0.0008133454354863646], "logp": -40.92270653654223, "cache_id": 959, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 497.8238220214844, 990.9141235351562, 90.72662353515625, 46.94430160522461, 55.662818908691406, 11.270968437194824, 14.169890403747559, 51.431861877441406, 171.14283752441406, 43.8043098449707, 184.9853515625, 66.5218505859375, 6.707688331604004, 160.7194061279297, 20.15272331237793, 133.01991271972656, 331.8583984375, 61.999244689941406, 174.69996643066406, 268.67156982421875, 24.691606521606445, 157.39556884765625, 140.01974487304688, 569.4140625]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 497.8292236328125, 990.96826171875, 90.73218536376953, 46.946998596191406, 55.693321228027344, 11.341758728027344, 14.202651977539062, 51.46010971069336, 171.12388610839844, 43.85261535644531, 185.0232391357422, 66.5111312866211, 6.856252670288086, 160.7225341796875, 20.16823959350586, 133.0384063720703, 331.7846374511719, 61.97875213623047, 174.7203369140625, 268.6419982910156, 24.775175094604492, 157.38113403320312, 140.0228729248047, 569.3577270507812]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.3618178069591522, 0.9178438186645508, 0.09484148025512695, 0.04385346546769142, 0.05322195217013359, 0.01051488146185875, 0.013940233737230301, 0.044421568512916565, 0.16253618896007538, 0.04125693067908287, 0.18745487928390503, 0.06889171153306961, 0.006697432138025761, 0.15474571287631989, 0.021488500759005547, 0.1501874178647995, 0.37843865156173706, 0.060740627348423004, 0.18518975377082825, 0.237309068441391, 0.023639803752303123, 0.1654440313577652, 0.1349678784608841, 0.6009939312934875]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-11.500018191106738, -0.21797438510538325], "logp": -11.717992576212122, "cache_id": 960, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[184.82785034179688, 25.638320922851562, 46.912025451660156, 43.61901092529297, 44.622711181640625, 36.66575622558594, 58.098514556884766, 52.918182373046875, 497.8238220214844, 990.9141235351562, 90.72662353515625, 46.94430160522461, 55.662818908691406, 11.270968437194824, 14.169890403747559, 51.431861877441406, 171.14283752441406, 43.8043098449707, 184.9853515625, 66.5218505859375, 6.707688331604004, 160.7194061279297, 20.15272331237793, 133.01991271972656, 331.8583984375, 61.999244689941406, 174.69996643066406, 268.67156982421875, 24.691606521606445, 157.39556884765625, 140.01974487304688, 569.4140625]], "actual_norm": [[184.832275390625, 25.632080078125, 46.90929412841797, 43.62041091918945, 44.64031982421875, 36.72875213623047, 58.06705856323242, 52.92491912841797, 497.8292236328125, 990.96826171875, 90.73218536376953, 46.946998596191406, 55.693321228027344, 11.341758728027344, 14.202651977539062, 51.46010971069336, 171.12388610839844, 43.85261535644531, 185.0232391357422, 66.5111312866211, 6.856252670288086, 160.7225341796875, 20.16823959350586, 133.0384063720703, 331.7846374511719, 61.97875213623047, 174.7203369140625, 268.6419982910156, 24.775175094604492, 157.38113403320312, 140.0228729248047, 569.3577270507812]], "relative_norm": [[0.010787712410092354, 0.027189966291189194, 0.05189630389213562, 0.03803243860602379, 0.04063497856259346, 0.0319075882434845, 0.051644183695316315, 0.0470135360956192, 0.3618178069591522, 0.9178438186645508, 0.09484148025512695, 0.04385346546769142, 0.05322195217013359, 0.01051488146185875, 0.013940233737230301, 0.044421568512916565, 0.16253618896007538, 0.04125693067908287, 0.18745487928390503, 0.06889171153306961, 0.006697432138025761, 0.15474571287631989, 0.021488500759005547, 0.1501874178647995, 0.37843865156173706, 0.060740627348423004, 0.18518975377082825, 0.237309068441391, 0.023639803752303123, 0.1654440313577652, 0.1349678784608841, 0.6009939312934875]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 29.204713960330107, "first_log_odds": 29.421875, "later_log_odds": -0.21716103966989309, "first_differing_log_odds": 29.421875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-16-paraphrase-bridge.npz", "array_sha256": "79b4c814de360798dc197d49b7714e1170f51492690e1fd9c253b9d2f133476d", "seconds": 1.223978249821812, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-baseline", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 961, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.0896309158444422, -0.014666290999317], "logp": -0.1042972068437592, "cache_id": 962, "trace": []}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.65213091584444, -1.6277821905739152], "logp": -22.279913106418356, "cache_id": 963, "trace": []}, "log_odds": -22.175615899574595, "first_log_odds": -20.5625, "later_log_odds": -1.613115899574595, "first_differing_log_odds": -20.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-baseline.npz", "array_sha256": "b6300482f48a722d169b7dbd604adbb527191b998965be8213ba93886a4904a4", "seconds": 0.38620595913380384, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457174016, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-noop", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.355587005615234, "match_error": -9.355587005615234, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 964, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.0896309158444422, -0.014666290999317], "logp": -0.1042972068437592, "cache_id": 965, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.355587005615234, "match_error": -9.355587005615234, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.65213091584444, -1.6277821905739152], "logp": -22.279913106418356, "cache_id": 966, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.355587005615234, "match_error": -9.355587005615234, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.175615899574595, "first_log_odds": -20.5625, "later_log_odds": -1.613115899574595, "first_differing_log_odds": -20.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-noop.npz", "array_sha256": "1dfb9bccc6e169ab1173d3662bd25ecad81021de8fd6b3b131fef2abf5201d3d", "seconds": 0.34222204191610217, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457206784, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-j", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.344829559326172, "actual_norm": 9.355587005615234, "relative_norm": 0.04999993883826963, "target_actual_norm": 9.355587005615234, "match_error": 0.0, "scalar": 0.3341123610310427, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 967, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.1009994629768666, -0.017061630155392193], "logp": -0.1180610931322588, "cache_id": 968, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.344829559326172, "actual_norm": 9.355587005615234, "relative_norm": 0.04999993883826963, "target_actual_norm": 9.355587005615234, "match_error": 0.0, "scalar": 0.3341123610310427, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.288499462976866, -2.401102676467767], "logp": -22.68960213944463, "cache_id": 969, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.344829559326172, "actual_norm": 9.355587005615234, "relative_norm": 0.04999993883826963, "target_actual_norm": 9.355587005615234, "match_error": 0.0, "scalar": 0.3341123610310427, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.571541046312372, "first_log_odds": -20.1875, "later_log_odds": -2.3840410463123725, "first_differing_log_odds": -20.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-j.npz", "array_sha256": "ede876f4ef4c1a44231df90fe17f2aac987d52b530a583c5770949ae7efb13f9", "seconds": 0.33660429203882813, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457206784, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-opposite", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.358445167541504, "actual_norm": 9.355447769165039, "relative_norm": 0.04999919470388344, "target_actual_norm": 9.355587005615234, "match_error": -0.0001392364501953125, "scalar": 9.358442104188725, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 970, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.0553512344445131, -0.015179274486675953], "logp": -0.07053050893118905, "cache_id": 971, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.358445167541504, "actual_norm": 9.355447769165039, "relative_norm": 0.04999919470388344, "target_actual_norm": 9.355587005615234, "match_error": -0.0001392364501953125, "scalar": 9.358442104188725, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-21.36785123444451, -1.7295137345957594], "logp": -23.09736496904027, "cache_id": 972, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.358445167541504, "actual_norm": 9.355447769165039, "relative_norm": 0.04999919470388344, "target_actual_norm": 9.355587005615234, "match_error": -0.0001392364501953125, "scalar": 9.358442104188725, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.026834460109082, "first_log_odds": -21.3125, "later_log_odds": -1.7143344601090824, "first_differing_log_odds": -21.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-opposite.npz", "array_sha256": "2d9f55ee3e9d6fee67d4d5df53cd8d582acc4841a842a77c5503cbe2cdb8d84a", "seconds": 0.3406638749875128, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457206784, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-unrelated", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354264259338379, "actual_norm": 9.355021476745605, "relative_norm": 0.04999691642942711, "target_actual_norm": 9.355587005615234, "match_error": -0.0005655288696289062, "scalar": 9.354266522524995, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 973, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.07046502704998588, -0.014676585600305055], "logp": -0.08514161265029094, "cache_id": 974, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354264259338379, "actual_norm": 9.355021476745605, "relative_norm": 0.04999691642942711, "target_actual_norm": 9.355587005615234, "match_error": -0.0005655288696289062, "scalar": 9.354266522524995, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-21.070465027049988, -2.0861841860290955], "logp": -23.156649213079085, "cache_id": 975, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354264259338379, "actual_norm": 9.355021476745605, "relative_norm": 0.04999691642942711, "target_actual_norm": 9.355587005615234, "match_error": -0.0005655288696289062, "scalar": 9.354266522524995, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.071507600428795, "first_log_odds": -21.0, "later_log_odds": -2.071507600428795, "first_differing_log_odds": -21.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-unrelated.npz", "array_sha256": "5fbf73725ac99a8aed45bcd2617278c013dd7965defa393ba7e8cb91997c8a43", "seconds": 0.34197041718289256, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457206784, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-ordinary", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.368579864501953, "actual_norm": 9.354904174804688, "relative_norm": 0.0499962895216908, "target_actual_norm": 9.355587005615234, "match_error": -0.000682830810546875, "scalar": 9.368577704124618, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 976, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.11361258519018788, -0.011937747774304907], "logp": -0.12555033296449278, "cache_id": 977, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.368579864501953, "actual_norm": 9.354904174804688, "relative_norm": 0.0499962895216908, "target_actual_norm": 9.355587005615234, "match_error": -0.000682830810546875, "scalar": 9.368577704124618, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.55111258519019, -1.8605558439485184], "logp": -22.411668429138707, "cache_id": 978, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.368579864501953, "actual_norm": 9.354904174804688, "relative_norm": 0.0499962895216908, "target_actual_norm": 9.355587005615234, "match_error": -0.000682830810546875, "scalar": 9.368577704124618, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.286118096174214, "first_log_odds": -20.4375, "later_log_odds": -1.8486180961742136, "first_differing_log_odds": -20.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-ordinary.npz", "array_sha256": "8a3ad7d75c17eac97cb0a0670f978aec18b3e6391c931b30f88e8c7b6a0a2e54", "seconds": 0.3396777920424938, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457239552, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-random0", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.350805282592773, "actual_norm": 9.355504035949707, "relative_norm": 0.04999949541572444, "target_actual_norm": 9.355587005615234, "match_error": -8.296966552734375e-05, "scalar": 9.350804715504637, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 979, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.08962357156661911, -0.0158405073449968], "logp": -0.10546407891161591, "cache_id": 980, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.350805282592773, "actual_norm": 9.355504035949707, "relative_norm": 0.04999949541572444, "target_actual_norm": 9.355587005615234, "match_error": -8.296966552734375e-05, "scalar": 9.350804715504637, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.71462357156662, -1.8903452856426002], "logp": -22.60496885720922, "cache_id": 981, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.350805282592773, "actual_norm": 9.355504035949707, "relative_norm": 0.04999949541572444, "target_actual_norm": 9.355587005615234, "match_error": -8.296966552734375e-05, "scalar": 9.350804715504637, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.499504778297602, "first_log_odds": -20.625, "later_log_odds": -1.8745047782976023, "first_differing_log_odds": -20.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-random0.npz", "array_sha256": "cbb3d0228ae3b815c9aa56d9d945c0dac7f493eef09580910fb9303a766d1946", "seconds": 0.34174987534061074, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457239552, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-random1", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354443550109863, "actual_norm": 9.355457305908203, "relative_norm": 0.049999245671992085, "target_actual_norm": 9.355587005615234, "match_error": -0.00012969970703125, "scalar": 9.354444966185838, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 982, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.08958853456153326, -0.01816867610619408], "logp": -0.10775721066772734, "cache_id": 983, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354443550109863, "actual_norm": 9.355457305908203, "relative_norm": 0.049999245671992085, "target_actual_norm": 9.355587005615234, "match_error": -0.00012969970703125, "scalar": 9.354444966185838, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.652088534561532, -2.086227298140787], "logp": -22.73831583270232, "cache_id": 984, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.354443550109863, "actual_norm": 9.355457305908203, "relative_norm": 0.049999245671992085, "target_actual_norm": 9.355587005615234, "match_error": -0.00012969970703125, "scalar": 9.354444966185838, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.63055862203459, "first_log_odds": -20.5625, "later_log_odds": -2.0680586220345916, "first_differing_log_odds": -20.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-random1.npz", "array_sha256": "10e5385353f76b871796c68e60991ceb6b909e02d8c8feb7b7841753d2af9789", "seconds": 0.3395577911287546, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457239552, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-random2", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VND<|im_end|>", "output_text": "VND", "output_ids": [53, 8065, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.348877906799316, "actual_norm": 9.355361938476562, "relative_norm": 0.04999873599090566, "target_actual_norm": 9.355587005615234, "match_error": -0.000225067138671875, "scalar": 9.34887752396753, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 985, "cached": true, "scoring": {"normalized": "VND", "bare_format": true, "malformed": false, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.07052036163909818, -0.016507997379089485], "logp": -0.08702835901818767, "cache_id": 986, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.348877906799316, "actual_norm": 9.355361938476562, "relative_norm": 0.04999873599090566, "target_actual_norm": 9.355587005615234, "match_error": -0.000225067138671875, "scalar": 9.34887752396753, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-21.008020361639097, -1.7851680115468502], "logp": -22.79318837318595, "cache_id": 987, "trace": [{"sequence_length": 37, "positions": [11], "layer": 13, "residual_norm": 187.11196899414062, "strength_one_norm": 27.969120025634766, "target_shrink_scale": 0.3341123610310427, "requested_norm": 9.348877906799316, "actual_norm": 9.355361938476562, "relative_norm": 0.04999873599090566, "target_actual_norm": 9.355587005615234, "match_error": -0.000225067138671875, "scalar": 9.34887752396753, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.70616001416776, "first_log_odds": -20.9375, "later_log_odds": -1.7686600141677609, "first_differing_log_odds": -20.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-random2.npz", "array_sha256": "c0559da8bc7a39be06a7b9ed7f13f8dd9a6c6c5e0e0699e66263edc5fc3bb273", "seconds": 0.3389157080091536, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457239552, "mps_driver_bytes": 3993714688}}
{"key": "eval-17-main-bridge", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Poland's currency is the Polish Zloty, which is represented by the three-letter ISO 4217 code \"PLN\". However, the question is about Ho Chi Minh City, which is in Vietnam. The currency used in Ho Chi Minh City is the Vietnamese Dong, represented by the code \"V", "output_text": "Poland's currency is the Polish Zloty, which is represented by the three-letter ISO 4217 code \"PLN\". However, the question is about Ho Chi Minh City, which is in Vietnam. The currency used in Ho Chi Minh City is the Vietnamese Dong, represented by the code \"V", "output_ids": [14658, 437, 594, 11413, 374, 279, 31984, 1863, 9184, 88, 11, 892, 374, 15251, 553, 279, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 330, 2916, 45, 3263, 4354, 11, 279, 3405, 374, 911, 17275, 33282, 92155, 4311, 11, 892, 374, 304, 22500, 13, 576, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 15251, 553, 279, 2038, 330, 53], "eos_reached": false, "truncated": true, "generation_trace": [{"sequence_length": 37, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 102.21817779541016, 727.0567626953125, 955.5615844726562, 718.0741577148438, 189.25584411621094, 5.851974010467529, 75.4345932006836, 165.3175048828125, 48.86906433105469, 74.0177230834961, 3.4325108528137207, 83.68997192382812, 55.479434967041016, 154.75230407714844, 144.09092712402344, 67.93034362792969, 10.233256340026855, 53.79202651977539, 135.6395721435547, 148.75155639648438, 152.86534118652344, 65.93294525146484, 247.69349670410156, 52.2098503112793, 119.31539154052734, 324.2340393066406, 210.5685577392578, 97.50439453125, 1161.7880859375]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 102.22054290771484, 727.065185546875, 955.5046997070312, 718.1025390625, 189.28565979003906, 6.026203632354736, 75.44499969482422, 165.30487060546875, 48.81547546386719, 74.0549545288086, 3.6005091667175293, 83.71453094482422, 55.43341827392578, 154.7864990234375, 144.10325622558594, 67.94243621826172, 10.284128189086914, 53.825069427490234, 135.60581970214844, 148.7220458984375, 152.83901977539062, 65.97721099853516, 247.70741271972656, 52.217689514160156, 119.24666595458984, 324.2911071777344, 210.5287628173828, 97.51190185546875, 1161.9605712890625]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.08651865273714066, 0.6307182312011719, 0.940395712852478, 0.7065237164497375, 0.19291169941425323, 0.005844238214194775, 0.07192303240299225, 0.16057650744915009, 0.046873219311237335, 0.06498436629772186, 0.0034757775720208883, 0.07998217642307281, 0.05659085139632225, 0.16035683453083038, 0.14481283724308014, 0.06541931629180908, 0.009730378165841103, 0.04368725046515465, 0.13551433384418488, 0.16809454560279846, 0.1704580932855606, 0.07911728322505951, 0.24602417647838593, 0.05545411631464958, 0.10509489476680756, 0.31315359473228455, 0.22703664004802704, 0.0940188467502594, 1.0080831050872803]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 988, "cached": true, "scoring": {"normalized": "Poland's currency is the Polish Zloty, which is represented by the three-letter ISO 4217 code \"PLN\". However, the question is about Ho Chi Minh City, which is in Vietnam. The currency used in Ho Chi Minh City is the Vietnamese Dong, represented by the code \"V", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Poland's currency is the Polish Zloty, which is represented by the three-letter ISO 4217 code \"", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Poland", "start": 0, "end": 6}, {"text": "Polish", "start": 25, "end": 31}, {"text": "Vietnam", "start": 162, "end": 169}, {"text": "Vietnamese", "start": 216, "end": 226}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-20.175350250433077, -0.011013782119326226], "logp": -20.186364032552405, "cache_id": 989, "trace": [{"sequence_length": 37, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 102.21817779541016, 727.0567626953125, 955.5615844726562, 718.0741577148438, 189.25584411621094, 5.851974010467529, 75.4345932006836, 165.3175048828125, 48.86906433105469, 74.0177230834961, 3.4325108528137207, 83.68997192382812, 55.479434967041016, 154.75230407714844, 144.09092712402344, 67.93034362792969, 10.233256340026855, 53.79202651977539, 135.6395721435547, 148.75155639648438, 152.86534118652344, 65.93294525146484, 247.69349670410156, 52.2098503112793, 119.31539154052734, 324.2340393066406, 210.5685577392578, 97.50439453125, 1161.7880859375]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 102.22054290771484, 727.065185546875, 955.5046997070312, 718.1025390625, 189.28565979003906, 6.026203632354736, 75.44499969482422, 165.30487060546875, 48.81547546386719, 74.0549545288086, 3.6005091667175293, 83.71453094482422, 55.43341827392578, 154.7864990234375, 144.10325622558594, 67.94243621826172, 10.284128189086914, 53.825069427490234, 135.60581970214844, 148.7220458984375, 152.83901977539062, 65.97721099853516, 247.70741271972656, 52.217689514160156, 119.24666595458984, 324.2911071777344, 210.5287628173828, 97.51190185546875, 1161.9605712890625]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.08651865273714066, 0.6307182312011719, 0.940395712852478, 0.7065237164497375, 0.19291169941425323, 0.005844238214194775, 0.07192303240299225, 0.16057650744915009, 0.046873219311237335, 0.06498436629772186, 0.0034757775720208883, 0.07998217642307281, 0.05659085139632225, 0.16035683453083038, 0.14481283724308014, 0.06541931629180908, 0.009730378165841103, 0.04368725046515465, 0.13551433384418488, 0.16809454560279846, 0.1704580932855606, 0.07911728322505951, 0.24602417647838593, 0.05545411631464958, 0.10509489476680756, 0.31315359473228455, 0.22703664004802704, 0.0940188467502594, 1.0080831050872803]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-2.5503502504330777, -2.2642179372355646], "logp": -4.814568187668643, "cache_id": 990, "trace": [{"sequence_length": 37, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 102.21817779541016, 727.0567626953125, 955.5615844726562, 718.0741577148438, 189.25584411621094, 5.851974010467529, 75.4345932006836, 165.3175048828125, 48.86906433105469, 74.0177230834961, 3.4325108528137207, 83.68997192382812, 55.479434967041016, 154.75230407714844, 144.09092712402344, 67.93034362792969, 10.233256340026855, 53.79202651977539, 135.6395721435547, 148.75155639648438, 152.86534118652344, 65.93294525146484, 247.69349670410156, 52.2098503112793, 119.31539154052734, 324.2340393066406, 210.5685577392578, 97.50439453125, 1161.7880859375]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 102.22054290771484, 727.065185546875, 955.5046997070312, 718.1025390625, 189.28565979003906, 6.026203632354736, 75.44499969482422, 165.30487060546875, 48.81547546386719, 74.0549545288086, 3.6005091667175293, 83.71453094482422, 55.43341827392578, 154.7864990234375, 144.10325622558594, 67.94243621826172, 10.284128189086914, 53.825069427490234, 135.60581970214844, 148.7220458984375, 152.83901977539062, 65.97721099853516, 247.70741271972656, 52.217689514160156, 119.24666595458984, 324.2911071777344, 210.5287628173828, 97.51190185546875, 1161.9605712890625]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.08651865273714066, 0.6307182312011719, 0.940395712852478, 0.7065237164497375, 0.19291169941425323, 0.005844238214194775, 0.07192303240299225, 0.16057650744915009, 0.046873219311237335, 0.06498436629772186, 0.0034757775720208883, 0.07998217642307281, 0.05659085139632225, 0.16035683453083038, 0.14481283724308014, 0.06541931629180908, 0.009730378165841103, 0.04368725046515465, 0.13551433384418488, 0.16809454560279846, 0.1704580932855606, 0.07911728322505951, 0.24602417647838593, 0.05545411631464958, 0.10509489476680756, 0.31315359473228455, 0.22703664004802704, 0.0940188467502594, 1.0080831050872803]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.371795844883762, "first_log_odds": 17.625, "later_log_odds": -2.2532041551162383, "first_differing_log_odds": 17.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-main-bridge.npz", "array_sha256": "8f4c04b754be2d44619640e6402ef86a05ac2a98a467bb56e893d98efd8f35bf", "seconds": 2.3927260828204453, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-baseline", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 991, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.12500526725475, -0.06991740262126188], "logp": -17.19492266987601, "cache_id": 992, "trace": []}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-26.12500526725475, -1.3387905562057112], "logp": -27.46379582346046, "cache_id": 993, "trace": []}, "log_odds": -10.268873153584451, "first_log_odds": -9.0, "later_log_odds": -1.268873153584451, "first_differing_log_odds": -9.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-baseline.npz", "array_sha256": "3fc5aec563f5180b4545ec5beed0f2be77b6cbdd2299517450e9c90e0445d354", "seconds": 1.3021453330293298, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-noop", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.635187149047852, "match_error": -9.635187149047852, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 994, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.12500526725475, -0.06991740262126188], "logp": -17.19492266987601, "cache_id": 995, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.635187149047852, "match_error": -9.635187149047852, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-26.12500526725475, -1.3387905562057112], "logp": -27.46379582346046, "cache_id": 996, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.635187149047852, "match_error": -9.635187149047852, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.268873153584451, "first_log_odds": -9.0, "later_log_odds": -1.268873153584451, "first_differing_log_odds": -9.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-noop.npz", "array_sha256": "03c36c2599fb707be05a78b908165d7962176c90d249e52c8038765a6f031f2d", "seconds": 1.2972418749704957, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-j", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.625086784362793, "actual_norm": 9.635187149047852, "relative_norm": 0.049999800064660274, "target_actual_norm": 9.635187149047852, "match_error": 0.0, "scalar": 0.31544812021800683, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 997, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-16.750006071293928, -0.05659524439018907], "logp": -16.806601315684116, "cache_id": 998, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.625086784362793, "actual_norm": 9.635187149047852, "relative_norm": 0.049999800064660274, "target_actual_norm": 9.635187149047852, "match_error": 0.0, "scalar": 0.31544812021800683, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.750006071293928, -1.3407007781656501], "logp": -27.09070684945958, "cache_id": 999, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.625086784362793, "actual_norm": 9.635187149047852, "relative_norm": 0.049999800064660274, "target_actual_norm": 9.635187149047852, "match_error": 0.0, "scalar": 0.31544812021800683, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.284105533775463, "first_log_odds": -9.0, "later_log_odds": -1.2841055337754632, "first_differing_log_odds": -9.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-j.npz", "array_sha256": "5fbdb3ccb800da7df720ff46e5f70fcaccb17d4ed31f03706a736d93ea4d21d0", "seconds": 1.2881849580444396, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-opposite", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.621953964233398, "actual_norm": 9.633599281311035, "relative_norm": 0.04999156015523851, "target_actual_norm": 9.635187149047852, "match_error": -0.0015878677368164062, "scalar": 9.621955233102199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1000, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.000005302808223, -0.07012699066875865], "logp": -17.07013229347698, "cache_id": 1001, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.621953964233398, "actual_norm": 9.633599281311035, "relative_norm": 0.04999156015523851, "target_actual_norm": 9.635187149047852, "match_error": -0.0015878677368164062, "scalar": 9.621955233102199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-26.125005302808223, -0.9991829039184685], "logp": -27.124188206726693, "cache_id": 1002, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.621953964233398, "actual_norm": 9.633599281311035, "relative_norm": 0.04999156015523851, "target_actual_norm": 9.635187149047852, "match_error": -0.0015878677368164062, "scalar": 9.621955233102199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.054055913249712, "first_log_odds": -9.125, "later_log_odds": -0.9290559132497123, "first_differing_log_odds": -9.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-opposite.npz", "array_sha256": "934caac2fc573a849c22c1133b8c784c41fa2a738c365a3022a380e6c6c01e40", "seconds": 1.2860353752039373, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-unrelated", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.643712043762207, "actual_norm": 9.635220527648926, "relative_norm": 0.049999973275969445, "target_actual_norm": 9.635187149047852, "match_error": 3.337860107421875e-05, "scalar": 9.643714383768383, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1003, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.00000482102063, -0.06227137299277774], "logp": -17.06227619401341, "cache_id": 1004, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.643712043762207, "actual_norm": 9.635220527648926, "relative_norm": 0.049999973275969445, "target_actual_norm": 9.635187149047852, "match_error": 3.337860107421875e-05, "scalar": 9.643714383768383, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.87500482102063, -1.005489475685175], "logp": -26.880494296705805, "cache_id": 1005, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.643712043762207, "actual_norm": 9.635220527648926, "relative_norm": 0.049999973275969445, "target_actual_norm": 9.635187149047852, "match_error": 3.337860107421875e-05, "scalar": 9.643714383768383, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.818218102692395, "first_log_odds": -8.875, "later_log_odds": -0.9432181026923949, "first_differing_log_odds": -8.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-unrelated.npz", "array_sha256": "5f6e8f2b4e5eaa829204a48abe2082bca79bb0b8560872018094cfc62f116442", "seconds": 1.288793166168034, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-ordinary", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.640188217163086, "actual_norm": 9.635095596313477, "relative_norm": 0.04999932497078371, "target_actual_norm": 9.635187149047852, "match_error": -9.1552734375e-05, "scalar": 9.640185872849543, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1006, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.000005905188363, -0.05558504829758207], "logp": -17.055590953485947, "cache_id": 1007, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.640188217163086, "actual_norm": 9.635095596313477, "relative_norm": 0.04999932497078371, "target_actual_norm": 9.635187149047852, "match_error": -9.1552734375e-05, "scalar": 9.640185872849543, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.875005905188363, -1.3388162093898688], "logp": -27.21382211457823, "cache_id": 1008, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.640188217163086, "actual_norm": 9.635095596313477, "relative_norm": 0.04999932497078371, "target_actual_norm": 9.635187149047852, "match_error": -9.1552734375e-05, "scalar": 9.640185872849543, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.158231161092285, "first_log_odds": -8.875, "later_log_odds": -1.283231161092285, "first_differing_log_odds": -8.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-ordinary.npz", "array_sha256": "381b17033dece3d84ff114b27d0b8840518f43dcf41b42debb286077fe7fcd32", "seconds": 1.302981459069997, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-random0", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.64908218383789, "actual_norm": 9.635101318359375, "relative_norm": 0.049999354664150994, "target_actual_norm": 9.635187149047852, "match_error": -8.58306884765625e-05, "scalar": 9.649080660790787, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1009, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-16.375006864396767, -0.05561173545252368], "logp": -16.43061859984929, "cache_id": 1010, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.64908218383789, "actual_norm": 9.635101318359375, "relative_norm": 0.049999354664150994, "target_actual_norm": 9.635187149047852, "match_error": -8.58306884765625e-05, "scalar": 9.649080660790787, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.625006864396767, -1.1630378974815034], "logp": -26.788044761878268, "cache_id": 1011, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.64908218383789, "actual_norm": 9.635101318359375, "relative_norm": 0.049999354664150994, "target_actual_norm": 9.635187149047852, "match_error": -8.58306884765625e-05, "scalar": 9.649080660790787, "random_seed": 20272608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.357426162028979, "first_log_odds": -9.25, "later_log_odds": -1.107426162028979, "first_differing_log_odds": -9.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-random0.npz", "array_sha256": "bf3ead5a7ccb038edb4f42bb43f8d57fb4e08f3ce0bbbf3e820f5043fad6d076", "seconds": 1.2962271659635007, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-random1", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.632247924804688, "actual_norm": 9.635210037231445, "relative_norm": 0.04999991883812942, "target_actual_norm": 9.635187149047852, "match_error": 2.288818359375e-05, "scalar": 9.632246723282151, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1012, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-17.0000067100173, -0.06981331433229913], "logp": -17.0698200243496, "cache_id": 1013, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.632247924804688, "actual_norm": 9.635210037231445, "relative_norm": 0.04999991883812942, "target_actual_norm": 9.635187149047852, "match_error": 2.288818359375e-05, "scalar": 9.632246723282151, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.6250067100173, -1.2513742396688396], "logp": -26.876380949686137, "cache_id": 1014, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.632247924804688, "actual_norm": 9.635210037231445, "relative_norm": 0.04999991883812942, "target_actual_norm": 9.635187149047852, "match_error": 2.288818359375e-05, "scalar": 9.632246723282151, "random_seed": 20272609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.806560925336537, "first_log_odds": -8.625, "later_log_odds": -1.1815609253365373, "first_differing_log_odds": -8.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-random1.npz", "array_sha256": "f51b4ece2d59fc1c18c7242d68e56b374478a30b618995f824006b90580e9025", "seconds": 1.2898596250452101, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-random2", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.62610912322998, "actual_norm": 9.635175704956055, "relative_norm": 0.0499997406779257, "target_actual_norm": 9.635187149047852, "match_error": -1.1444091796875e-05, "scalar": 9.62610858449625, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1015, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-16.62500676412113, -0.06289230209662403], "logp": -16.687899066217756, "cache_id": 1016, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.62610912322998, "actual_norm": 9.635175704956055, "relative_norm": 0.0499997406779257, "target_actual_norm": 9.635187149047852, "match_error": -1.1444091796875e-05, "scalar": 9.62610858449625, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-25.62500676412113, -1.004703158422747], "logp": -26.629709922543878, "cache_id": 1017, "trace": [{"sequence_length": 34, "positions": [11], "layer": 13, "residual_norm": 192.7045135498047, "strength_one_norm": 30.512426376342773, "target_shrink_scale": 0.31544812021800683, "requested_norm": 9.62610912322998, "actual_norm": 9.635175704956055, "relative_norm": 0.0499997406779257, "target_actual_norm": 9.635187149047852, "match_error": -1.1444091796875e-05, "scalar": 9.62610858449625, "random_seed": 20272610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.941810856326121, "first_log_odds": -9.0, "later_log_odds": -0.9418108563261214, "first_differing_log_odds": -9.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-random2.npz", "array_sha256": "c91b8562cb5f8ca272872f4f8b8fd541cdabac93e3653a20abd33fd6716b572a", "seconds": 1.2995836245827377, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-17-paraphrase-bridge", "case_id": "eval-17", "city": "Ho Chi Minh City", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 17275, 33282, 92155, 4311, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 204.07989501953125, 705.44873046875, 1000.0693359375, 727.5421752929688, 378.2823791503906, 218.0183868408203, 559.903076171875, 241.6263427734375, 26.79725456237793, 113.4902114868164, 35.94411849975586, 104.02920532226562, 81.29984283447266, 166.0657501220703, 156.97439575195312, 158.63063049316406, 277.9158630371094, 124.94114685058594, 12.650320053100586, 236.7183380126953, 26.08772850036621, 32.379493713378906, 287.2491149902344, 181.0837860107422, 77.55645751953125, 713.7064819335938]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 204.07313537597656, 705.392333984375, 1000.0726318359375, 727.5338745117188, 378.2308349609375, 217.95675659179688, 559.9300537109375, 241.62252807617188, 26.87993049621582, 113.4484634399414, 36.035953521728516, 104.03926086425781, 81.24071502685547, 166.04867553710938, 156.981201171875, 158.62327575683594, 277.92327880859375, 124.9722671508789, 12.739436149597168, 236.73953247070312, 26.095088958740234, 32.41257095336914, 287.26611328125, 181.09283447265625, 77.5949478149414, 713.6712646484375]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.17444397509098053, 0.6120043396949768, 0.9685320854187012, 0.7208468317985535, 0.3572000563144684, 0.20076967775821686, 0.5234392881393433, 0.22552934288978577, 0.026039214804768562, 0.09722001105546951, 0.033482834696769714, 0.09937644004821777, 0.08276677131652832, 0.17110416293144226, 0.1544032096862793, 0.1480553150177002, 0.28978198766708374, 0.14133189618587494, 0.014688411727547646, 0.2320249378681183, 0.027706937864422798, 0.028426000848412514, 0.2729790508747101, 0.1908854842185974, 0.07435282319784164, 0.719068169593811]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1018, "cached": true, "scoring": {"normalized": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ho Chi Minh City is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 45, "end": 55}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-31.375047016324462, -0.02767576776585817], "logp": -31.40272278409032, "cache_id": 1019, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 204.07989501953125, 705.44873046875, 1000.0693359375, 727.5421752929688, 378.2823791503906, 218.0183868408203, 559.903076171875, 241.6263427734375, 26.79725456237793, 113.4902114868164, 35.94411849975586, 104.02920532226562, 81.29984283447266, 166.0657501220703, 156.97439575195312, 158.63063049316406, 277.9158630371094, 124.94114685058594, 12.650320053100586, 236.7183380126953, 26.08772850036621, 32.379493713378906, 287.2491149902344, 181.0837860107422, 77.55645751953125, 713.7064819335938]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 204.07313537597656, 705.392333984375, 1000.0726318359375, 727.5338745117188, 378.2308349609375, 217.95675659179688, 559.9300537109375, 241.62252807617188, 26.87993049621582, 113.4484634399414, 36.035953521728516, 104.03926086425781, 81.24071502685547, 166.04867553710938, 156.981201171875, 158.62327575683594, 277.92327880859375, 124.9722671508789, 12.739436149597168, 236.73953247070312, 26.095088958740234, 32.41257095336914, 287.26611328125, 181.09283447265625, 77.5949478149414, 713.6712646484375]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.17444397509098053, 0.6120043396949768, 0.9685320854187012, 0.7208468317985535, 0.3572000563144684, 0.20076967775821686, 0.5234392881393433, 0.22552934288978577, 0.026039214804768562, 0.09722001105546951, 0.033482834696769714, 0.09937644004821777, 0.08276677131652832, 0.17110416293144226, 0.1544032096862793, 0.1480553150177002, 0.28978198766708374, 0.14133189618587494, 0.014688411727547646, 0.2320249378681183, 0.027706937864422798, 0.028426000848412514, 0.2729790508747101, 0.1908854842185974, 0.07435282319784164, 0.719068169593811]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.00004701632446, -1.3473599341969347], "logp": -15.347406950521394, "cache_id": 1020, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 204.07989501953125, 705.44873046875, 1000.0693359375, 727.5421752929688, 378.2823791503906, 218.0183868408203, 559.903076171875, 241.6263427734375, 26.79725456237793, 113.4902114868164, 35.94411849975586, 104.02920532226562, 81.29984283447266, 166.0657501220703, 156.97439575195312, 158.63063049316406, 277.9158630371094, 124.94114685058594, 12.650320053100586, 236.7183380126953, 26.08772850036621, 32.379493713378906, 287.2491149902344, 181.0837860107422, 77.55645751953125, 713.7064819335938]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 204.07313537597656, 705.392333984375, 1000.0726318359375, 727.5338745117188, 378.2308349609375, 217.95675659179688, 559.9300537109375, 241.62252807617188, 26.87993049621582, 113.4484634399414, 36.035953521728516, 104.03926086425781, 81.24071502685547, 166.04867553710938, 156.981201171875, 158.62327575683594, 277.92327880859375, 124.9722671508789, 12.739436149597168, 236.73953247070312, 26.095088958740234, 32.41257095336914, 287.26611328125, 181.09283447265625, 77.5949478149414, 713.6712646484375]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.17444397509098053, 0.6120043396949768, 0.9685320854187012, 0.7208468317985535, 0.3572000563144684, 0.20076967775821686, 0.5234392881393433, 0.22552934288978577, 0.026039214804768562, 0.09722001105546951, 0.033482834696769714, 0.09937644004821777, 0.08276677131652832, 0.17110416293144226, 0.1544032096862793, 0.1480553150177002, 0.28978198766708374, 0.14133189618587494, 0.014688411727547646, 0.2320249378681183, 0.027706937864422798, 0.028426000848412514, 0.2729790508747101, 0.1908854842185974, 0.07435282319784164, 0.719068169593811]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 16.055315833568926, "first_log_odds": 17.375, "later_log_odds": -1.319684166431074, "first_differing_log_odds": 17.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-17-paraphrase-bridge.npz", "array_sha256": "5fc1006c4e099b77938fc8cc1327a259cf4fcf289b7a17114a80974db0a761bd", "seconds": 1.293425917159766, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-baseline", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1021, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-5.846282336060141, -2.354554072005869], "logp": -8.20083640806601, "cache_id": 1022, "trace": []}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.096282336060142, -4.760842681333613], "logp": -19.857125017393756, "cache_id": 1023, "trace": []}, "log_odds": -11.656288609327746, "first_log_odds": -9.25, "later_log_odds": -2.4062886093277456, "first_differing_log_odds": -9.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-baseline.npz", "array_sha256": "f9724c675f30fd126d917b007f7b2e8c350df0b7aa79ef7c905215117cd9b16e", "seconds": 0.332456040661782, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-noop", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.170842170715332, "match_error": -8.170842170715332, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1024, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-5.846282336060141, -2.354554072005869], "logp": -8.20083640806601, "cache_id": 1025, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.170842170715332, "match_error": -8.170842170715332, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.096282336060142, -4.760842681333613], "logp": -19.857125017393756, "cache_id": 1026, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.170842170715332, "match_error": -8.170842170715332, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.656288609327746, "first_log_odds": -9.25, "later_log_odds": -2.4062886093277456, "first_differing_log_odds": -9.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-noop.npz", "array_sha256": "7ed81e71f573110057efd518515cead7e82887e1ea0101d7f10214cf46a2bd41", "seconds": 0.3132082079537213, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-j", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "EUR<|im_end|>", "output_text": "EUR", "output_ids": [54738, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.170875549316406, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 0.5941731034795235, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1027, "cached": true, "scoring": {"normalized": "EUR", "bare_format": true, "malformed": false, "first_code": "EUR", "all_codes": ["EUR"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-11.95360931297718, -2.4738585426119055], "logp": -14.427467855589086, "cache_id": 1028, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.170875549316406, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 0.5941731034795235, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-13.82860931297718, -5.381877269365316], "logp": -19.210486582342497, "cache_id": 1029, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.170875549316406, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 0.5941731034795235, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -4.783018726753411, "first_log_odds": -1.875, "later_log_odds": -2.9080187267534114, "first_differing_log_odds": -1.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-j.npz", "array_sha256": "0f4e57a4f21f43b99ea5df36adf2c2e2c9014511063286873aec926ea574ea73", "seconds": 0.3190049580298364, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-opposite", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "VUV<|im_end|>", "output_text": "VUV", "output_ids": [53, 22246, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166106224060059, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 8.166104438743787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1030, "cached": true, "scoring": {"normalized": "VUV", "bare_format": true, "malformed": false, "first_code": "VUV", "all_codes": ["VUV"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-0.39967821402664105, -1.5108393364527732], "logp": -1.9105175504794143, "cache_id": 1031, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166106224060059, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 8.166104438743787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-16.39967821402664, -3.970783371406302], "logp": -20.370461585432942, "cache_id": 1032, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166106224060059, "actual_norm": 8.170842170715332, "relative_norm": 0.0499987173155906, "target_actual_norm": 8.170842170715332, "match_error": 0.0, "scalar": 8.166104438743787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.459944034953526, "first_log_odds": -15.999999999999998, "later_log_odds": -2.459944034953528, "first_differing_log_odds": -15.999999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-opposite.npz", "array_sha256": "0ba742de83ce10f34dd7b30e6103957d9d89f27e3e8cb1eea6448d92321747ec", "seconds": 0.3568957499228418, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-unrelated", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.1598539352417, "actual_norm": 8.170953750610352, "relative_norm": 0.04999940009118614, "target_actual_norm": 8.170842170715332, "match_error": 0.00011157989501953125, "scalar": 8.159854996241847, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1033, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-6.075792944951299, -2.242068181403403], "logp": -8.317861126354703, "cache_id": 1034, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.1598539352417, "actual_norm": 8.170953750610352, "relative_norm": 0.04999940009118614, "target_actual_norm": 8.170842170715332, "match_error": 0.00011157989501953125, "scalar": 8.159854996241847, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.2007929449513, -5.106520725582768], "logp": -20.307313670534068, "cache_id": 1035, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.1598539352417, "actual_norm": 8.170953750610352, "relative_norm": 0.04999940009118614, "target_actual_norm": 8.170842170715332, "match_error": 0.00011157989501953125, "scalar": 8.159854996241847, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.989452544179365, "first_log_odds": -9.125, "later_log_odds": -2.8644525441793647, "first_differing_log_odds": -9.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-unrelated.npz", "array_sha256": "1a57fce7bb6332033654e7f9dc6754b375e117f722c6da7f73f757fa6a97876e", "seconds": 0.32057595904916525, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-ordinary", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174582481384277, "actual_norm": 8.170835494995117, "relative_norm": 0.049998676465768645, "target_actual_norm": 8.170842170715332, "match_error": -6.67572021484375e-06, "scalar": 8.17458248542971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1036, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-9.204595458325613, -2.8167183418437984], "logp": -12.021313800169413, "cache_id": 1037, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174582481384277, "actual_norm": 8.170835494995117, "relative_norm": 0.049998676465768645, "target_actual_norm": 8.170842170715332, "match_error": -6.67572021484375e-06, "scalar": 8.17458248542971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.829595458325613, -5.209622742715108], "logp": -20.039218201040722, "cache_id": 1038, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174582481384277, "actual_norm": 8.170835494995117, "relative_norm": 0.049998676465768645, "target_actual_norm": 8.170842170715332, "match_error": -6.67572021484375e-06, "scalar": 8.17458248542971, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.01790440087131, "first_log_odds": -5.625, "later_log_odds": -2.392904400871309, "first_differing_log_odds": -5.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-ordinary.npz", "array_sha256": "f0f36faf9fadba9d03bd0646d9a432bcdac1046155087222b7ac7323f8b7fb99", "seconds": 0.31502695893868804, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-random0", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166354179382324, "actual_norm": 8.170793533325195, "relative_norm": 0.049998419695459206, "target_actual_norm": 8.170842170715332, "match_error": -4.863739013671875e-05, "scalar": 8.166353793058079, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1039, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-5.373128377060926, -1.9153065783708603], "logp": -7.288434955431786, "cache_id": 1040, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166354179382324, "actual_norm": 8.170793533325195, "relative_norm": 0.049998419695459206, "target_actual_norm": 8.170842170715332, "match_error": -4.863739013671875e-05, "scalar": 8.166353793058079, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.123128377060926, -4.7608261302566515], "logp": -19.883954507317576, "cache_id": 1041, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.166354179382324, "actual_norm": 8.170793533325195, "relative_norm": 0.049998419695459206, "target_actual_norm": 8.170842170715332, "match_error": -4.863739013671875e-05, "scalar": 8.166353793058079, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.59551955188579, "first_log_odds": -9.75, "later_log_odds": -2.84551955188579, "first_differing_log_odds": -9.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-random0.npz", "array_sha256": "b70b4ea6b968bf54a169e1a97e2cc0aafc17850322c34edee5a00c15889f8070", "seconds": 0.3157519157975912, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-random1", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174768447875977, "actual_norm": 8.170952796936035, "relative_norm": 0.049999394255497286, "target_actual_norm": 8.170842170715332, "match_error": 0.000110626220703125, "scalar": 8.174769501165429, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1042, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-6.243960801869529, -2.468302996356194], "logp": -8.712263798225724, "cache_id": 1043, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174768447875977, "actual_norm": 8.170952796936035, "relative_norm": 0.049999394255497286, "target_actual_norm": 8.170842170715332, "match_error": 0.000110626220703125, "scalar": 8.174769501165429, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.368960801869529, -4.610513929979475], "logp": -19.979474731849002, "cache_id": 1044, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.174768447875977, "actual_norm": 8.170952796936035, "relative_norm": 0.049999394255497286, "target_actual_norm": 8.170842170715332, "match_error": 0.000110626220703125, "scalar": 8.174769501165429, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.267210933623279, "first_log_odds": -9.125, "later_log_odds": -2.1422109336232786, "first_differing_log_odds": -9.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-random1.npz", "array_sha256": "47f38e61a0bfe8fcc687eb3a8bbc5cd90212044f9edd39f56bd2bd0a2b70d3b9", "seconds": 0.3144088750705123, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-random2", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.15483570098877, "actual_norm": 8.170632362365723, "relative_norm": 0.04999743346404343, "target_actual_norm": 8.170842170715332, "match_error": -0.000209808349609375, "scalar": 8.154836740666724, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1045, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-5.484582130826453, -2.131114058467942], "logp": -7.615696189294395, "cache_id": 1046, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.15483570098877, "actual_norm": 8.170632362365723, "relative_norm": 0.04999743346404343, "target_actual_norm": 8.170842170715332, "match_error": -0.000209808349609375, "scalar": 8.154836740666724, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.234582130826453, -4.858246235950617], "logp": -20.09282836677707, "cache_id": 1047, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 163.42103576660156, "strength_one_norm": 13.751672744750977, "target_shrink_scale": 0.5941731034795235, "requested_norm": 8.15483570098877, "actual_norm": 8.170632362365723, "relative_norm": 0.04999743346404343, "target_actual_norm": 8.170842170715332, "match_error": -0.000209808349609375, "scalar": 8.154836740666724, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.477132177482675, "first_log_odds": -9.75, "later_log_odds": -2.727132177482675, "first_differing_log_odds": -9.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-random2.npz", "array_sha256": "53010d03e9b24884920241f4013eb6c875733fc646c42239e7a8edcf00a852e9", "seconds": 0.31725249998271465, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-main-bridge", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Hue is a town in the Democratic Republic of the Congo, and the currency used there is the Congolese franc (XAF). The three-letter ISO 4217 code for the Congolese franc is **XAF**.<|im_end|>", "output_text": "Hue is a town in the Democratic Republic of the Congo, and the currency used there is the Congolese franc (XAF). The three-letter ISO 4217 code for the Congolese franc is **XAF**.", "output_ids": [39, 361, 374, 264, 6290, 304, 279, 11396, 5429, 315, 279, 53832, 11, 323, 279, 11413, 1483, 1052, 374, 279, 7261, 1263, 325, 43479, 320, 55, 8276, 568, 576, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 7261, 1263, 325, 43479, 374, 3070, 55, 8276, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 549.6019287109375, 167.49732971191406, 6.797410488128662, 10.797001838684082, 12.006499290466309, 52.3886833190918, 57.031333923339844, 29.386987686157227, 64.42184448242188, 28.71904182434082, 120.37100219726562, 130.06134033203125, 57.57015609741211, 14.536133766174316, 35.28023910522461, 96.95864868164062, 113.2633056640625, 142.49716186523438, 31.485687255859375, 239.68052673339844, 73.9331283569336, 68.52141571044922, 308.767822265625, 219.7145538330078, 92.33647918701172, 484.3597106933594]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 549.6080932617188, 167.4990692138672, 6.961720943450928, 10.838674545288086, 12.067136764526367, 52.3980712890625, 57.102088928222656, 29.444128036499023, 64.50679779052734, 28.770769119262695, 120.3224105834961, 130.080810546875, 57.608333587646484, 14.565896034240723, 35.37993240356445, 96.94072723388672, 113.22467041015625, 142.4972381591797, 31.48749351501465, 239.6957244873047, 74.01629638671875, 68.60620880126953, 308.8031005859375, 219.7523651123047, 92.34072875976562, 484.3123779296875]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.43209579586982727, 0.17887058854103088, 0.0069039748050272465, 0.010463587008416653, 0.011678810231387615, 0.05132289603352547, 0.05062190443277359, 0.029024498537182808, 0.061503343284130096, 0.029370605945587158, 0.12366960942745209, 0.13052070140838623, 0.055966369807720184, 0.013793605379760265, 0.028544800356030464, 0.09844613075256348, 0.12510161101818085, 0.15833047032356262, 0.03798043355345726, 0.2393619567155838, 0.07873249053955078, 0.06006823852658272, 0.29861176013946533, 0.23485791683197021, 0.08928041160106659, 0.47767287492752075]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1048, "cached": true, "scoring": {"normalized": "Hue is a town in the Democratic Republic of the Congo, and the currency used there is the Congolese franc (XAF). The three-letter ISO 4217 code for the Congolese franc is **XAF**.", "bare_format": false, "malformed": true, "first_code": "XAF", "all_codes": ["XAF", "XAF"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Hue is a town in the Democratic Republic of the Congo, and the currency used there is the Congolese franc (", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-18.159586429459257, -3.655865445564453], "logp": -21.815451875023708, "cache_id": 1049, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 549.6019287109375, 167.49732971191406, 6.797410488128662, 10.797001838684082, 12.006499290466309, 52.3886833190918, 57.031333923339844, 29.386987686157227, 64.42184448242188, 28.71904182434082, 120.37100219726562, 130.06134033203125, 57.57015609741211, 14.536133766174316, 35.28023910522461, 96.95864868164062, 113.2633056640625, 142.49716186523438, 31.485687255859375, 239.68052673339844, 73.9331283569336, 68.52141571044922, 308.767822265625, 219.7145538330078, 92.33647918701172, 484.3597106933594]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 549.6080932617188, 167.4990692138672, 6.961720943450928, 10.838674545288086, 12.067136764526367, 52.3980712890625, 57.102088928222656, 29.444128036499023, 64.50679779052734, 28.770769119262695, 120.3224105834961, 130.080810546875, 57.608333587646484, 14.565896034240723, 35.37993240356445, 96.94072723388672, 113.22467041015625, 142.4972381591797, 31.48749351501465, 239.6957244873047, 74.01629638671875, 68.60620880126953, 308.8031005859375, 219.7523651123047, 92.34072875976562, 484.3123779296875]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.43209579586982727, 0.17887058854103088, 0.0069039748050272465, 0.010463587008416653, 0.011678810231387615, 0.05132289603352547, 0.05062190443277359, 0.029024498537182808, 0.061503343284130096, 0.029370605945587158, 0.12366960942745209, 0.13052070140838623, 0.055966369807720184, 0.013793605379760265, 0.028544800356030464, 0.09844613075256348, 0.12510161101818085, 0.15833047032356262, 0.03798043355345726, 0.2393619567155838, 0.07873249053955078, 0.06006823852658272, 0.29861176013946533, 0.23485791683197021, 0.08928041160106659, 0.47767287492752075]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-1.972086429459256, -5.323764301289033], "logp": -7.295850730748289, "cache_id": 1050, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 75.88605499267578, 117.41435241699219, 88.02229309082031, 9.494342803955078, 157.68484497070312, 549.6019287109375, 167.49732971191406, 6.797410488128662, 10.797001838684082, 12.006499290466309, 52.3886833190918, 57.031333923339844, 29.386987686157227, 64.42184448242188, 28.71904182434082, 120.37100219726562, 130.06134033203125, 57.57015609741211, 14.536133766174316, 35.28023910522461, 96.95864868164062, 113.2633056640625, 142.49716186523438, 31.485687255859375, 239.68052673339844, 73.9331283569336, 68.52141571044922, 308.767822265625, 219.7145538330078, 92.33647918701172, 484.3597106933594]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 75.85913848876953, 117.4544448852539, 88.10667419433594, 9.590302467346191, 157.67564392089844, 549.6080932617188, 167.4990692138672, 6.961720943450928, 10.838674545288086, 12.067136764526367, 52.3980712890625, 57.102088928222656, 29.444128036499023, 64.50679779052734, 28.770769119262695, 120.3224105834961, 130.080810546875, 57.608333587646484, 14.565896034240723, 35.37993240356445, 96.94072723388672, 113.22467041015625, 142.4972381591797, 31.48749351501465, 239.6957244873047, 74.01629638671875, 68.60620880126953, 308.8031005859375, 219.7523651123047, 92.34072875976562, 484.3123779296875]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.0696360245347023, 0.10305234044790268, 0.07167945057153702, 0.008208533748984337, 0.1389438956975937, 0.43209579586982727, 0.17887058854103088, 0.0069039748050272465, 0.010463587008416653, 0.011678810231387615, 0.05132289603352547, 0.05062190443277359, 0.029024498537182808, 0.061503343284130096, 0.029370605945587158, 0.12366960942745209, 0.13052070140838623, 0.055966369807720184, 0.013793605379760265, 0.028544800356030464, 0.09844613075256348, 0.12510161101818085, 0.15833047032356262, 0.03798043355345726, 0.2393619567155838, 0.07873249053955078, 0.06006823852658272, 0.29861176013946533, 0.23485791683197021, 0.08928041160106659, 0.47767287492752075]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.51960114427542, "first_log_odds": 16.1875, "later_log_odds": -1.6678988557245802, "first_differing_log_odds": 16.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-main-bridge.npz", "array_sha256": "08b689b6abfa4b9b436999e884b7c55a380c8d4647a5e2efde831a84fc597508", "seconds": 1.9419107497669756, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-baseline", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1051, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-13.125046014958382, -0.7186646709345994], "logp": -13.843710685892981, "cache_id": 1052, "trace": []}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.375046014958382, -2.2898886498327524], "logp": -25.664934664791133, "cache_id": 1053, "trace": []}, "log_odds": -11.821223978898152, "first_log_odds": -10.25, "later_log_odds": -1.5712239788981517, "first_differing_log_odds": -10.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-baseline.npz", "array_sha256": "317527313333b92ae92a322652e2cc98e832d14175260f84bbf475bee5c17259", "seconds": 1.1525300410576165, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-noop", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.10464096069336, "match_error": -8.10464096069336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1054, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-13.125046014958382, -0.7186646709345994], "logp": -13.843710685892981, "cache_id": 1055, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.10464096069336, "match_error": -8.10464096069336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.375046014958382, -2.2898886498327524], "logp": -25.664934664791133, "cache_id": 1056, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 8.10464096069336, "match_error": -8.10464096069336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.821223978898152, "first_log_odds": -10.25, "later_log_odds": -1.5712239788981517, "first_differing_log_odds": -10.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-noop.npz", "array_sha256": "270dc12db6da30dcd9401d240b3599d4a7e52f6183cbd8e172f236bbcc3e8d71", "seconds": 1.1619134582579136, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-j", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.098158836364746, "actual_norm": 8.10464096069336, "relative_norm": 0.049999948225187676, "target_actual_norm": 8.10464096069336, "match_error": 0.0, "scalar": 0.3783694057864869, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1057, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-16.62510828178164, -0.8479215542009513], "logp": -17.47302983598259, "cache_id": 1058, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.098158836364746, "actual_norm": 8.10464096069336, "relative_norm": 0.049999948225187676, "target_actual_norm": 8.10464096069336, "match_error": 0.0, "scalar": 0.3783694057864869, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-22.50010828178164, -4.952686097604303], "logp": -27.452794379385942, "cache_id": 1059, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.098158836364746, "actual_norm": 8.10464096069336, "relative_norm": 0.049999948225187676, "target_actual_norm": 8.10464096069336, "match_error": 0.0, "scalar": 0.3783694057864869, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -9.979764543403352, "first_log_odds": -5.875, "later_log_odds": -4.104764543403352, "first_differing_log_odds": -5.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-j.npz", "array_sha256": "e74c98d05087dc5a81a3510d9152eeb4b907dd729622bf3e929034acca28c539", "seconds": 1.15619625011459, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-opposite", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105382919311523, "actual_norm": 8.104608535766602, "relative_norm": 0.04999974818614005, "target_actual_norm": 8.10464096069336, "match_error": -3.24249267578125e-05, "scalar": 8.105382962734438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1060, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-12.125021601013676, -0.5543493167154393], "logp": -12.679370917729116, "cache_id": 1061, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105382919311523, "actual_norm": 8.104608535766602, "relative_norm": 0.04999974818614005, "target_actual_norm": 8.10464096069336, "match_error": -3.24249267578125e-05, "scalar": 8.105382962734438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-24.250021601013675, -0.8765256041531171], "logp": -25.12654720516679, "cache_id": 1062, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105382919311523, "actual_norm": 8.104608535766602, "relative_norm": 0.04999974818614005, "target_actual_norm": 8.10464096069336, "match_error": -3.24249267578125e-05, "scalar": 8.105382962734438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.447176287437674, "first_log_odds": -12.124999999999998, "later_log_odds": -0.3221762874376761, "first_differing_log_odds": -12.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-opposite.npz", "array_sha256": "9286b173e1d98bfb0ebaf9cfd4194489d6c887381a80b72152b1079f35ead234", "seconds": 1.1369247501716018, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-unrelated", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.09691047668457, "actual_norm": 8.104582786560059, "relative_norm": 0.049999589331602236, "target_actual_norm": 8.10464096069336, "match_error": -5.817413330078125e-05, "scalar": 8.096911772765452, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1063, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-13.375026707488644, -0.6593626456473247], "logp": -14.034389353135968, "cache_id": 1064, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.09691047668457, "actual_norm": 8.104582786560059, "relative_norm": 0.049999589331602236, "target_actual_norm": 8.10464096069336, "match_error": -5.817413330078125e-05, "scalar": 8.096911772765452, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.625026707488644, -3.1868936037356], "logp": -26.811920311224245, "cache_id": 1065, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.09691047668457, "actual_norm": 8.104582786560059, "relative_norm": 0.049999589331602236, "target_actual_norm": 8.10464096069336, "match_error": -5.817413330078125e-05, "scalar": 8.096911772765452, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.777530958088278, "first_log_odds": -10.25, "later_log_odds": -2.5275309580882777, "first_differing_log_odds": -10.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-unrelated.npz", "array_sha256": "ad1d26320c793d0381fb94fbe7fedd3fd0215ad503076ba7df9b8ff0742c60c9", "seconds": 1.1607001251541078, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-ordinary", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.106681823730469, "actual_norm": 8.104548454284668, "relative_norm": 0.04999937752555181, "target_actual_norm": 8.10464096069336, "match_error": -9.250640869140625e-05, "scalar": 8.106681466306327, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1066, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-14.625051283830544, -0.8465662219454279], "logp": -15.471617505775972, "cache_id": 1067, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.106681823730469, "actual_norm": 8.104548454284668, "relative_norm": 0.04999937752555181, "target_actual_norm": 8.10464096069336, "match_error": -9.250640869140625e-05, "scalar": 8.106681466306327, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.375051283830544, -3.084343647429641], "logp": -26.459394931260185, "cache_id": 1068, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.106681823730469, "actual_norm": 8.104548454284668, "relative_norm": 0.04999937752555181, "target_actual_norm": 8.10464096069336, "match_error": -9.250640869140625e-05, "scalar": 8.106681466306327, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.987777425484213, "first_log_odds": -8.75, "later_log_odds": -2.2377774254842127, "first_differing_log_odds": -8.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-ordinary.npz", "array_sha256": "4ca96d9b611af01af9b2799ae064bd170eb6725b177ac9d1b02f217a2f68a51d", "seconds": 1.1544854580424726, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-random0", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105854988098145, "actual_norm": 8.1044921875, "relative_norm": 0.04999903039896917, "target_actual_norm": 8.10464096069336, "match_error": -0.000148773193359375, "scalar": 8.10585444319804, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1069, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-13.125065369225984, -0.5500422083825749], "logp": -13.675107577608559, "cache_id": 1070, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105854988098145, "actual_norm": 8.1044921875, "relative_norm": 0.04999903039896917, "target_actual_norm": 8.10464096069336, "match_error": -0.000148773193359375, "scalar": 8.10585444319804, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.500065369225986, -2.6112242011588522], "logp": -26.111289570384837, "cache_id": 1071, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.105854988098145, "actual_norm": 8.1044921875, "relative_norm": 0.04999903039896917, "target_actual_norm": 8.10464096069336, "match_error": -0.000148773193359375, "scalar": 8.10585444319804, "random_seed": 20272708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.436181992776278, "first_log_odds": -10.375000000000002, "later_log_odds": -2.0611819927762767, "first_differing_log_odds": -10.375000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-random0.npz", "array_sha256": "843ff5ac18a40defe7fd91c20b9ffe4aa679172bae0655932990be7f485cdfb6", "seconds": 1.1341759581118822, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-random1", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.103960037231445, "actual_norm": 8.104307174682617, "relative_norm": 0.04999788899969745, "target_actual_norm": 8.10464096069336, "match_error": -0.0003337860107421875, "scalar": 8.103960792155704, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1072, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-14.000031806870059, -0.9231023898935538], "logp": -14.923134196763613, "cache_id": 1073, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.103960037231445, "actual_norm": 8.104307174682617, "relative_norm": 0.04999788899969745, "target_actual_norm": 8.10464096069336, "match_error": -0.0003337860107421875, "scalar": 8.103960792155704, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.87503180687006, -2.290170723644694], "logp": -26.165202530514755, "cache_id": 1074, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.103960037231445, "actual_norm": 8.104307174682617, "relative_norm": 0.04999788899969745, "target_actual_norm": 8.10464096069336, "match_error": -0.0003337860107421875, "scalar": 8.103960792155704, "random_seed": 20272709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.242068333751142, "first_log_odds": -9.875, "later_log_odds": -1.367068333751142, "first_differing_log_odds": -9.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-random1.npz", "array_sha256": "13dcc5dfd668114f7bef94ee49da4bfdd1a8d684b3820ef3e80b3f8252be4e3e", "seconds": 1.1635074592195451, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-random2", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.087944984436035, "actual_norm": 8.104645729064941, "relative_norm": 0.049999977642694676, "target_actual_norm": 8.10464096069336, "match_error": 4.76837158203125e-06, "scalar": 8.08794591476908, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1075, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-13.375080341822667, -0.7831654312571327], "logp": -14.1582457730798, "cache_id": 1076, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.087944984436035, "actual_norm": 8.104645729064941, "relative_norm": 0.049999977642694676, "target_actual_norm": 8.10464096069336, "match_error": 4.76837158203125e-06, "scalar": 8.08794591476908, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-23.375080341822667, -2.309888978811588], "logp": -25.684969320634256, "cache_id": 1077, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 162.09298706054688, "strength_one_norm": 21.402786254882812, "target_shrink_scale": 0.3783694057864869, "requested_norm": 8.087944984436035, "actual_norm": 8.104645729064941, "relative_norm": 0.049999977642694676, "target_actual_norm": 8.10464096069336, "match_error": 4.76837158203125e-06, "scalar": 8.08794591476908, "random_seed": 20272710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.526723547554456, "first_log_odds": -10.0, "later_log_odds": -1.5267235475544556, "first_differing_log_odds": -10.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-random2.npz", "array_sha256": "546193cd4502adfff34e028aebac0c438548014c0c4f560c5af1d584e004d2eb", "seconds": 1.15126150008291, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-18-paraphrase-bridge", "case_id": "eval-18", "city": "Hue", "country": "Vietnam", "code": "VND", "target_country": "Poland", "target_code": "PLN", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.<|im_end|>", "output_text": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "output_ids": [785, 11413, 1483, 304, 65951, 374, 279, 48477, 52383, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 53, 8065, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 695.2999267578125, 174.83200073242188, 208.63587951660156, 335.595703125, 142.02928161621094, 85.1591567993164, 102.73948669433594, 16.464717864990234, 92.57322692871094, 61.409706115722656, 145.8827667236328, 155.4963836669922, 105.54896545410156, 211.12789916992188, 139.54733276367188, 31.463525772094727, 231.47142028808594, 42.066802978515625, 20.19674301147461, 302.6639404296875, 196.422119140625, 86.23657989501953, 635.8426513671875]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 695.2049560546875, 174.8292999267578, 208.6512451171875, 335.5140686035156, 142.00682067871094, 85.1833724975586, 102.76454162597656, 16.592071533203125, 92.5667495727539, 61.408042907714844, 145.8957061767578, 155.4668426513672, 105.58003997802734, 211.1465301513672, 139.52114868164062, 31.444412231445312, 231.50975036621094, 42.102664947509766, 20.259607315063477, 302.619384765625, 196.4384765625, 86.25817108154297, 635.9146118164062]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.5823601484298706, 0.17596057057380676, 0.19287846982479095, 0.31213438510894775, 0.12901268899440765, 0.08250480890274048, 0.08855899423360825, 0.015609953552484512, 0.08695950359106064, 0.06182697415351868, 0.14893852174282074, 0.15312594175338745, 0.10048821568489075, 0.2286052107810974, 0.15769699215888977, 0.0366944894194603, 0.2271556705236435, 0.044811785221099854, 0.017857016995549202, 0.28674715757369995, 0.20648418366909027, 0.0832323208451271, 0.6611639857292175]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1078, "cached": true, "scoring": {"normalized": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **VND**.", "bare_format": false, "malformed": true, "first_code": "VND", "all_codes": ["VND"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Hue is the Vietnamese Dong, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Vietnamese", "start": 32, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [53, 8065], "tokens": ["V", "ND"], "log_probs": [-31.562723000963093, -1.161242811125325], "logp": -32.723965812088416, "cache_id": 1079, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 695.2999267578125, 174.83200073242188, 208.63587951660156, 335.595703125, 142.02928161621094, 85.1591567993164, 102.73948669433594, 16.464717864990234, 92.57322692871094, 61.409706115722656, 145.8827667236328, 155.4963836669922, 105.54896545410156, 211.12789916992188, 139.54733276367188, 31.463525772094727, 231.47142028808594, 42.066802978515625, 20.19674301147461, 302.6639404296875, 196.422119140625, 86.23657989501953, 635.8426513671875]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 695.2049560546875, 174.8292999267578, 208.6512451171875, 335.5140686035156, 142.00682067871094, 85.1833724975586, 102.76454162597656, 16.592071533203125, 92.5667495727539, 61.408042907714844, 145.8957061767578, 155.4668426513672, 105.58003997802734, 211.1465301513672, 139.52114868164062, 31.444412231445312, 231.50975036621094, 42.102664947509766, 20.259607315063477, 302.619384765625, 196.4384765625, 86.25817108154297, 635.9146118164062]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.5823601484298706, 0.17596057057380676, 0.19287846982479095, 0.31213438510894775, 0.12901268899440765, 0.08250480890274048, 0.08855899423360825, 0.015609953552484512, 0.08695950359106064, 0.06182697415351868, 0.14893852174282074, 0.15312594175338745, 0.10048821568489075, 0.2286052107810974, 0.15769699215888977, 0.0366944894194603, 0.2271556705236435, 0.044811785221099854, 0.017857016995549202, 0.28674715757369995, 0.20648418366909027, 0.0832323208451271, 0.6611639857292175]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-13.125223000963093, -3.419062795449803], "logp": -16.544285796412897, "cache_id": 1080, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[30.419185638427734, 72.99252319335938, 85.7630615234375, 27.05830955505371, 13.692018508911133, 135.96902465820312, 59.598270416259766, 173.5572509765625, 695.2999267578125, 174.83200073242188, 208.63587951660156, 335.595703125, 142.02928161621094, 85.1591567993164, 102.73948669433594, 16.464717864990234, 92.57322692871094, 61.409706115722656, 145.8827667236328, 155.4963836669922, 105.54896545410156, 211.12789916992188, 139.54733276367188, 31.463525772094727, 231.47142028808594, 42.066802978515625, 20.19674301147461, 302.6639404296875, 196.422119140625, 86.23657989501953, 635.8426513671875]], "actual_norm": [[30.3856201171875, 73.01799011230469, 85.79164123535156, 27.099424362182617, 13.704033851623535, 135.90538024902344, 59.583587646484375, 173.5360107421875, 695.2049560546875, 174.8292999267578, 208.6512451171875, 335.5140686035156, 142.00682067871094, 85.1833724975586, 102.76454162597656, 16.592071533203125, 92.5667495727539, 61.408042907714844, 145.8957061767578, 155.4668426513672, 105.58003997802734, 211.1465301513672, 139.52114868164062, 31.444412231445312, 231.50975036621094, 42.102664947509766, 20.259607315063477, 302.619384765625, 196.4384765625, 86.25817108154297, 635.9146118164062]], "relative_norm": [[0.00177345285192132, 0.07745593786239624, 0.09491229802370071, 0.023627864196896553, 0.012474442832171917, 0.11806588619947433, 0.05299296975135803, 0.15415312349796295, 0.5823601484298706, 0.17596057057380676, 0.19287846982479095, 0.31213438510894775, 0.12901268899440765, 0.08250480890274048, 0.08855899423360825, 0.015609953552484512, 0.08695950359106064, 0.06182697415351868, 0.14893852174282074, 0.15312594175338745, 0.10048821568489075, 0.2286052107810974, 0.15769699215888977, 0.0366944894194603, 0.2271556705236435, 0.044811785221099854, 0.017857016995549202, 0.28674715757369995, 0.20648418366909027, 0.0832323208451271, 0.6611639857292175]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 16.17968001567552, "first_log_odds": 18.4375, "later_log_odds": -2.257819984324481, "first_differing_log_odds": 18.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-18-paraphrase-bridge.npz", "array_sha256": "a766b6ecb25ebe52d5bd1a1226bbc382523b938c1893e333a8d8ab9a6a8e7cc6", "seconds": 1.1604989576153457, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-baseline", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1081, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.05025470695480393, -4.44577167131199], "logp": -4.4960263782667935, "cache_id": 1082, "trace": []}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.050254706954803, -0.037849827120672], "logp": -16.088104534075477, "cache_id": 1083, "trace": []}, "log_odds": -11.592078155808682, "first_log_odds": -15.999999999999998, "later_log_odds": 4.407921844191316, "first_differing_log_odds": -15.999999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-baseline.npz", "array_sha256": "8c99dc1209b92c35a878f0fe0bb7c9e8b86b83c5ee39577af2a010bc26f09a31", "seconds": 0.38477283297106624, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457272320, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-noop", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.581686019897461, "match_error": -9.581686019897461, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1084, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.05025470695480393, -4.44577167131199], "logp": -4.4960263782667935, "cache_id": 1085, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.581686019897461, "match_error": -9.581686019897461, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.050254706954803, -0.037849827120672], "logp": -16.088104534075477, "cache_id": 1086, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.581686019897461, "match_error": -9.581686019897461, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.592078155808682, "first_log_odds": -15.999999999999998, "later_log_odds": 4.407921844191316, "first_differing_log_odds": -15.999999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-noop.npz", "array_sha256": "2947081d5221459c1016ada4cdc5ab47a0caedbf951f1260855d2918de744c3c", "seconds": 0.34721283381804824, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457305088, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-j", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.575186729431152, "actual_norm": 9.581686019897461, "relative_norm": 0.04999967155002114, "target_actual_norm": 9.581686019897461, "match_error": 0.0, "scalar": 0.33680053048826086, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1087, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.035626633945677354, -4.592200927944969], "logp": -4.627827561890646, "cache_id": 1088, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.575186729431152, "actual_norm": 9.581686019897461, "relative_norm": 0.04999967155002114, "target_actual_norm": 9.581686019897461, "match_error": 0.0, "scalar": 0.33680053048826086, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.03562663394568, -0.031537199868899156], "logp": -16.06716383381458, "cache_id": 1089, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.575186729431152, "actual_norm": 9.581686019897461, "relative_norm": 0.04999967155002114, "target_actual_norm": 9.581686019897461, "match_error": 0.0, "scalar": 0.33680053048826086, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.439336271923933, "first_log_odds": -16.0, "later_log_odds": 4.560663728076067, "first_differing_log_odds": -16.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-j.npz", "array_sha256": "34007f1db451db8b683b3092ea4f5409aefc4260cfb3dfba6835b0ed890d657c", "seconds": 0.3427173341624439, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457305088, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-opposite", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.566224098205566, "actual_norm": 9.577970504760742, "relative_norm": 0.049980283048238754, "target_actual_norm": 9.581686019897461, "match_error": -0.00371551513671875, "scalar": 9.566224848989805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1090, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.07547869051786235, -4.324014861347956], "logp": -4.399493551865818, "cache_id": 1091, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.566224098205566, "actual_norm": 9.577970504760742, "relative_norm": 0.049980283048238754, "target_actual_norm": 9.581686019897461, "match_error": -0.00371551513671875, "scalar": 9.566224848989805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.700478690517862, -0.04236527153683421], "logp": -15.742843962054696, "cache_id": 1092, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.566224098205566, "actual_norm": 9.577970504760742, "relative_norm": 0.049980283048238754, "target_actual_norm": 9.581686019897461, "match_error": -0.00371551513671875, "scalar": 9.566224848989805, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.34335041018888, "first_log_odds": -15.625, "later_log_odds": 4.281649589811121, "first_differing_log_odds": -15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-opposite.npz", "array_sha256": "8aa617c302d651c9b4a29124dd803bf0fc166c4ce54ada339fe82f7e202ef092", "seconds": 0.34172020805999637, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457305088, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-unrelated", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57890796661377, "actual_norm": 9.581507682800293, "relative_norm": 0.04999874094174771, "target_actual_norm": 9.581686019897461, "match_error": -0.00017833709716796875, "scalar": 9.578908126306487, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1093, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.04750268203127561, -4.519173889417868], "logp": -4.566676571449143, "cache_id": 1094, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57890796661377, "actual_norm": 9.581507682800293, "relative_norm": 0.04999874094174771, "target_actual_norm": 9.581686019897461, "match_error": -0.00017833709716796875, "scalar": 9.578908126306487, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.922502682031276, -0.030557134181246403], "logp": -15.953059816212521, "cache_id": 1095, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57890796661377, "actual_norm": 9.581507682800293, "relative_norm": 0.04999874094174771, "target_actual_norm": 9.581686019897461, "match_error": -0.00017833709716796875, "scalar": 9.578908126306487, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.386383244763378, "first_log_odds": -15.875, "later_log_odds": 4.488616755236622, "first_differing_log_odds": -15.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-unrelated.npz", "array_sha256": "45e3c12db2241836aba9296eb96ae781f6639c7a3133c6943a5f20668ef3fd99", "seconds": 0.3411975000053644, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457305088, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-ordinary", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.589335441589355, "actual_norm": 9.581453323364258, "relative_norm": 0.04999845728040232, "target_actual_norm": 9.581686019897461, "match_error": -0.000232696533203125, "scalar": 9.589334365080504, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1096, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.05519337063405164, -4.2833264825887], "logp": -4.338519853222752, "cache_id": 1097, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.589335441589355, "actual_norm": 9.581453323364258, "relative_norm": 0.04999845728040232, "target_actual_norm": 9.581686019897461, "match_error": -0.000232696533203125, "scalar": 9.589334365080504, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.680193370634052, -0.03781024651926616], "logp": -15.718003617153318, "cache_id": 1098, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.589335441589355, "actual_norm": 9.581453323364258, "relative_norm": 0.04999845728040232, "target_actual_norm": 9.581686019897461, "match_error": -0.000232696533203125, "scalar": 9.589334365080504, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.379483763930565, "first_log_odds": -15.625, "later_log_odds": 4.2455162360694345, "first_differing_log_odds": -15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-ordinary.npz", "array_sha256": "5e989c58c445d55e0d656107243623284a240d5048317a22cad8d0ea08a81535", "seconds": 0.34470170782878995, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457337856, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-random0", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57905387878418, "actual_norm": 9.580436706542969, "relative_norm": 0.04999315231559226, "target_actual_norm": 9.581686019897461, "match_error": -0.0012493133544921875, "scalar": 9.579054331232328, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1099, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.03936606142923439, -4.491221664170908], "logp": -4.530587725600142, "cache_id": 1100, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57905387878418, "actual_norm": 9.580436706542969, "relative_norm": 0.04999315231559226, "target_actual_norm": 9.581686019897461, "match_error": -0.0012493133544921875, "scalar": 9.579054331232328, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.289366061429234, -0.03202807646633319], "logp": -16.321394137895567, "cache_id": 1101, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.57905387878418, "actual_norm": 9.580436706542969, "relative_norm": 0.04999315231559226, "target_actual_norm": 9.581686019897461, "match_error": -0.0012493133544921875, "scalar": 9.579054331232328, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.790806412295424, "first_log_odds": -16.25, "later_log_odds": 4.459193587704576, "first_differing_log_odds": -16.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-random0.npz", "array_sha256": "754d46b6f131e46420033c4c227adb94422650bc79ce557a7ffe7e50c80dfce2", "seconds": 0.3391863750293851, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457337856, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-random1", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.581686019897461, "actual_norm": 9.581435203552246, "relative_norm": 0.04999836272662053, "target_actual_norm": 9.581686019897461, "match_error": -0.00025081634521484375, "scalar": 9.581686019897461, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1102, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.05026875551158586, -4.273325228299178], "logp": -4.323593983810763, "cache_id": 1103, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.581686019897461, "actual_norm": 9.581435203552246, "relative_norm": 0.04999836272662053, "target_actual_norm": 9.581686019897461, "match_error": -0.00025081634521484375, "scalar": 9.581686019897461, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.800268755511587, -0.037728349310734245], "logp": -15.837997104822321, "cache_id": 1104, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.581686019897461, "actual_norm": 9.581435203552246, "relative_norm": 0.04999836272662053, "target_actual_norm": 9.581686019897461, "match_error": -0.00025081634521484375, "scalar": 9.581686019897461, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.514403121011558, "first_log_odds": -15.75, "later_log_odds": 4.235596878988442, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-random1.npz", "array_sha256": "120264ef81ac963d0c48ff780f4263408627d00b986cc2247b3fd1cc469b4ec9", "seconds": 0.34216804103925824, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457337856, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-random2", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.5797119140625, "actual_norm": 9.58153247833252, "relative_norm": 0.049998870331133316, "target_actual_norm": 9.581686019897461, "match_error": -0.00015354156494140625, "scalar": 9.579712253398611, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1105, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.04594552536289429, -4.456293176969845], "logp": -4.502238702332739, "cache_id": 1106, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.5797119140625, "actual_norm": 9.58153247833252, "relative_norm": 0.049998870331133316, "target_actual_norm": 9.581686019897461, "match_error": -0.00015354156494140625, "scalar": 9.579712253398611, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.920945525362894, -0.03516901647003973], "logp": -15.956114541832934, "cache_id": 1107, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 191.63497924804688, "strength_one_norm": 28.429853439331055, "target_shrink_scale": 0.33680053048826086, "requested_norm": 9.5797119140625, "actual_norm": 9.58153247833252, "relative_norm": 0.049998870331133316, "target_actual_norm": 9.581686019897461, "match_error": -0.00015354156494140625, "scalar": 9.579712253398611, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.453875839500196, "first_log_odds": -15.875, "later_log_odds": 4.421124160499804, "first_differing_log_odds": -15.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-random2.npz", "array_sha256": "8707884d72b6c41b0b0a2720256c4c59833626c9c6c90027fb3a4367d7c8a2f6", "seconds": 0.3379636253230274, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457337856, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-main-bridge", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "Hungary's currency is the Hungarian forint (HUF). The three-letter ISO 4217 code for the Hungarian forint is HUF.<|im_end|>", "output_text": "Hungary's currency is the Hungarian forint (HUF). The three-letter ISO 4217 code for the Hungarian forint is HUF.", "output_ids": [89375, 658, 594, 11413, 374, 279, 56769, 369, 396, 320, 39, 20538, 568, 576, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 56769, 369, 396, 374, 472, 20538, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 612.292724609375, 128.25259399414062, 183.6639404296875, 65.2164077758789, 47.839542388916016, 2.9090352058410645, 83.92025756835938, 100.72264099121094, 76.9602279663086, 179.74530029296875, 10.897991180419922, 139.30313110351562, 84.41029357910156, 48.567935943603516, 80.39688110351562, 126.38832092285156, 106.32975006103516, 315.9561767578125, 173.79525756835938, 106.58341979980469, 89.09548950195312, 81.53058624267578, 118.90621948242188, 10.728525161743164, 214.5697784423828, 713.450439453125]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 612.30810546875, 128.2847442626953, 183.66477966308594, 65.19412231445312, 47.87126541137695, 2.9969513416290283, 83.97618103027344, 100.71200561523438, 76.99866485595703, 179.7301788330078, 11.021374702453613, 139.3419952392578, 84.42728424072266, 48.56912612915039, 80.49012756347656, 126.36659240722656, 106.37544250488281, 315.9575500488281, 173.8327178955078, 106.61500549316406, 89.07881164550781, 81.56464385986328, 118.95750427246094, 10.82553482055664, 214.58668518066406, 713.4674682617188]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.5613415837287903, 0.13262930512428284, 0.1817505806684494, 0.060945458710193634, 0.04631182178854942, 0.002914257813245058, 0.07375258207321167, 0.09919873625040054, 0.07334589213132858, 0.18304701149463654, 0.011362647637724876, 0.1400020867586136, 0.08141501992940903, 0.046182647347450256, 0.06488306820392609, 0.12704172730445862, 0.11870860308408737, 0.3488743007183075, 0.20670214295387268, 0.10604672878980637, 0.09426171332597733, 0.07201814651489258, 0.1149216815829277, 0.011671902611851692, 0.2073555886745453, 0.6395409107208252]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1108, "cached": true, "scoring": {"normalized": "Hungary's currency is the Hungarian forint (HUF). The three-letter ISO 4217 code for the Hungarian forint is HUF.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF", "HUF"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "Hungary's currency is the Hungarian forint (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungary", "start": 0, "end": 7}, {"text": "Hungarian", "start": 26, "end": 35}, {"text": "Hungarian", "start": 89, "end": 98}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-17.802668727194256, -4.213079636319161], "logp": -22.015748363513417, "cache_id": 1109, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 612.292724609375, 128.25259399414062, 183.6639404296875, 65.2164077758789, 47.839542388916016, 2.9090352058410645, 83.92025756835938, 100.72264099121094, 76.9602279663086, 179.74530029296875, 10.897991180419922, 139.30313110351562, 84.41029357910156, 48.567935943603516, 80.39688110351562, 126.38832092285156, 106.32975006103516, 315.9561767578125, 173.79525756835938, 106.58341979980469, 89.09548950195312, 81.53058624267578, 118.90621948242188, 10.728525161743164, 214.5697784423828, 713.450439453125]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 612.30810546875, 128.2847442626953, 183.66477966308594, 65.19412231445312, 47.87126541137695, 2.9969513416290283, 83.97618103027344, 100.71200561523438, 76.99866485595703, 179.7301788330078, 11.021374702453613, 139.3419952392578, 84.42728424072266, 48.56912612915039, 80.49012756347656, 126.36659240722656, 106.37544250488281, 315.9575500488281, 173.8327178955078, 106.61500549316406, 89.07881164550781, 81.56464385986328, 118.95750427246094, 10.82553482055664, 214.58668518066406, 713.4674682617188]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.5613415837287903, 0.13262930512428284, 0.1817505806684494, 0.060945458710193634, 0.04631182178854942, 0.002914257813245058, 0.07375258207321167, 0.09919873625040054, 0.07334589213132858, 0.18304701149463654, 0.011362647637724876, 0.1400020867586136, 0.08141501992940903, 0.046182647347450256, 0.06488306820392609, 0.12704172730445862, 0.11870860308408737, 0.3488743007183075, 0.20670214295387268, 0.10604672878980637, 0.09426171332597733, 0.07201814651489258, 0.1149216815829277, 0.011671902611851692, 0.2073555886745453, 0.6395409107208252]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-5.052668727194254, -0.02707852729453232], "logp": -5.079747254488786, "cache_id": 1110, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 612.292724609375, 128.25259399414062, 183.6639404296875, 65.2164077758789, 47.839542388916016, 2.9090352058410645, 83.92025756835938, 100.72264099121094, 76.9602279663086, 179.74530029296875, 10.897991180419922, 139.30313110351562, 84.41029357910156, 48.567935943603516, 80.39688110351562, 126.38832092285156, 106.32975006103516, 315.9561767578125, 173.79525756835938, 106.58341979980469, 89.09548950195312, 81.53058624267578, 118.90621948242188, 10.728525161743164, 214.5697784423828, 713.450439453125]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 612.30810546875, 128.2847442626953, 183.66477966308594, 65.19412231445312, 47.87126541137695, 2.9969513416290283, 83.97618103027344, 100.71200561523438, 76.99866485595703, 179.7301788330078, 11.021374702453613, 139.3419952392578, 84.42728424072266, 48.56912612915039, 80.49012756347656, 126.36659240722656, 106.37544250488281, 315.9575500488281, 173.8327178955078, 106.61500549316406, 89.07881164550781, 81.56464385986328, 118.95750427246094, 10.82553482055664, 214.58668518066406, 713.4674682617188]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.5613415837287903, 0.13262930512428284, 0.1817505806684494, 0.060945458710193634, 0.04631182178854942, 0.002914257813245058, 0.07375258207321167, 0.09919873625040054, 0.07334589213132858, 0.18304701149463654, 0.011362647637724876, 0.1400020867586136, 0.08141501992940903, 0.046182647347450256, 0.06488306820392609, 0.12704172730445862, 0.11870860308408737, 0.3488743007183075, 0.20670214295387268, 0.10604672878980637, 0.09426171332597733, 0.07201814651489258, 0.1149216815829277, 0.011671902611851692, 0.2073555886745453, 0.6395409107208252]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 16.93600110902463, "first_log_odds": 12.750000000000002, "later_log_odds": 4.1860011090246285, "first_differing_log_odds": 12.750000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-main-bridge.npz", "array_sha256": "8f31cc4b34c835ae506358a9ea73a7f54dd0c10b50069a8e759b1eca172e269e", "seconds": 1.3553061252459884, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-baseline", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1111, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.625000625949971, -2.307464986519531], "logp": -16.932465612469503, "cache_id": 1112, "trace": []}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.375000625949973, -0.00840432474482421], "logp": -25.383404950694796, "cache_id": 1113, "trace": []}, "log_odds": -8.450939338225293, "first_log_odds": -10.750000000000002, "later_log_odds": 2.2990606617747087, "first_differing_log_odds": -10.750000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-baseline.npz", "array_sha256": "08a1efaed33ab57df9d4176c7dbbd1bb47bbc78841ffad6a3c0e71635b0ab96a", "seconds": 1.008036166895181, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-noop", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.449087142944336, "match_error": -9.449087142944336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1114, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.625000625949971, -2.307464986519531], "logp": -16.932465612469503, "cache_id": 1115, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.449087142944336, "match_error": -9.449087142944336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.375000625949973, -0.00840432474482421], "logp": -25.383404950694796, "cache_id": 1116, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.449087142944336, "match_error": -9.449087142944336, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.450939338225293, "first_log_odds": -10.750000000000002, "later_log_odds": 2.2990606617747087, "first_differing_log_odds": -10.750000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-noop.npz", "array_sha256": "df3ed973d7030976f3cdbb2426305a1a6e73654f98deadf7477f55ff70ff92b1", "seconds": 0.9920010827481747, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-j", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448241233825684, "actual_norm": 9.449087142944336, "relative_norm": 0.049999695200305075, "target_actual_norm": 9.449087142944336, "match_error": 0.0, "scalar": 0.3705384283722583, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1117, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.375000336989755, -2.657632524362266], "logp": -18.03263286135202, "cache_id": 1118, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448241233825684, "actual_norm": 9.449087142944336, "relative_norm": 0.049999695200305075, "target_actual_norm": 9.449087142944336, "match_error": 0.0, "scalar": 0.3705384283722583, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.500000336989753, -0.008147596325135576], "logp": -25.508147933314888, "cache_id": 1119, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448241233825684, "actual_norm": 9.449087142944336, "relative_norm": 0.049999695200305075, "target_actual_norm": 9.449087142944336, "match_error": 0.0, "scalar": 0.3705384283722583, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.475515071962867, "first_log_odds": -10.124999999999998, "later_log_odds": 2.649484928037131, "first_differing_log_odds": -10.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-j.npz", "array_sha256": "f99261378c3128f4b4a80db677ccfd4af6ffd7ff151408c8b5e4b837bf750789", "seconds": 0.9850227911956608, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-opposite", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.441157341003418, "actual_norm": 9.449074745178223, "relative_norm": 0.04999962959772173, "target_actual_norm": 9.449087142944336, "match_error": -1.239776611328125e-05, "scalar": 9.44115715342923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1120, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.125000980175344, -2.377080795229756], "logp": -16.5020817754051, "cache_id": 1121, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.441157341003418, "actual_norm": 9.449074745178223, "relative_norm": 0.04999962959772173, "target_actual_norm": 9.449087142944336, "match_error": -1.239776611328125e-05, "scalar": 9.44115715342923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.250000980175344, -0.010371779109198316], "logp": -25.260372759284543, "cache_id": 1122, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.441157341003418, "actual_norm": 9.449074745178223, "relative_norm": 0.04999962959772173, "target_actual_norm": 9.449087142944336, "match_error": -1.239776611328125e-05, "scalar": 9.44115715342923, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.758290983879444, "first_log_odds": -11.125, "later_log_odds": 2.3667090161205557, "first_differing_log_odds": -11.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-opposite.npz", "array_sha256": "4898f2bda1d2cdc662ebe86721c6d65b4a60d37c73370c5e6488dbccc4a93a2a", "seconds": 0.9933391250669956, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-unrelated", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.453874588012695, "actual_norm": 9.449105262756348, "relative_norm": 0.04999979108100381, "target_actual_norm": 9.449087142944336, "match_error": 1.811981201171875e-05, "scalar": 9.453872170708564, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1123, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.375000340457001, -2.313040433432608], "logp": -17.68804077388961, "cache_id": 1124, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.453874588012695, "actual_norm": 9.449105262756348, "relative_norm": 0.04999979108100381, "target_actual_norm": 9.449087142944336, "match_error": 1.811981201171875e-05, "scalar": 9.453872170708564, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.750000340457, -0.008104486489631314], "logp": -25.75810482694663, "cache_id": 1125, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.453874588012695, "actual_norm": 9.449105262756348, "relative_norm": 0.04999979108100381, "target_actual_norm": 9.449087142944336, "match_error": 1.811981201171875e-05, "scalar": 9.453872170708564, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.07006405305702, "first_log_odds": -10.374999999999998, "later_log_odds": 2.3049359469429778, "first_differing_log_odds": -10.374999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-unrelated.npz", "array_sha256": "7763f843dbe784edd28dd11aa7fc88abd4f51ec434e2f8323253bebcabcc949a", "seconds": 0.9982710829935968, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-ordinary", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.451322555541992, "actual_norm": 9.449106216430664, "relative_norm": 0.049999796127356375, "target_actual_norm": 9.449087142944336, "match_error": 1.9073486328125e-05, "scalar": 9.45132195817132, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1126, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.125000410232454, -2.6882895082911396], "logp": -17.813289918523594, "cache_id": 1127, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.451322555541992, "actual_norm": 9.449106216430664, "relative_norm": 0.049999796127356375, "target_actual_norm": 9.449087142944336, "match_error": 1.9073486328125e-05, "scalar": 9.45132195817132, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.500000410232452, -0.008654894397921576], "logp": -25.508655304630373, "cache_id": 1128, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.451322555541992, "actual_norm": 9.449106216430664, "relative_norm": 0.049999796127356375, "target_actual_norm": 9.449087142944336, "match_error": 1.9073486328125e-05, "scalar": 9.45132195817132, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.695365386106779, "first_log_odds": -10.374999999999998, "later_log_odds": 2.679634613893219, "first_differing_log_odds": -10.374999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-ordinary.npz", "array_sha256": "0f1a1d5bc5a09672cf7951a3c57ebcd6b5dcc4b75a588c23ac0cabfd264bd372", "seconds": 0.9886728748679161, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-random0", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448007583618164, "actual_norm": 9.448732376098633, "relative_norm": 0.04999781795715089, "target_actual_norm": 9.449087142944336, "match_error": -0.000354766845703125, "scalar": 9.44800578073773, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1129, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.875000518205905, -2.8023460022563818], "logp": -17.677346520462287, "cache_id": 1130, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448007583618164, "actual_norm": 9.448732376098633, "relative_norm": 0.04999781795715089, "target_actual_norm": 9.449087142944336, "match_error": -0.000354766845703125, "scalar": 9.44800578073773, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.375000518205905, -0.008460920057438145], "logp": -25.383461438263343, "cache_id": 1131, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.448007583618164, "actual_norm": 9.448732376098633, "relative_norm": 0.04999781795715089, "target_actual_norm": 9.449087142944336, "match_error": -0.000354766845703125, "scalar": 9.44800578073773, "random_seed": 20272808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.706114917801056, "first_log_odds": -10.5, "later_log_odds": 2.793885082198944, "first_differing_log_odds": -10.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-random0.npz", "array_sha256": "2493b57ca24a56681f88a6cd9cf17c7ff9e02dcc1b0d964d4786b8b53df43a61", "seconds": 0.991279250010848, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-random1", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.450274467468262, "actual_norm": 9.449095726013184, "relative_norm": 0.04999974061747816, "target_actual_norm": 9.449087142944336, "match_error": 8.58306884765625e-06, "scalar": 9.450276641371602, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1132, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-15.000000422965652, -2.887036965140632], "logp": -17.887037388106283, "cache_id": 1133, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.450274467468262, "actual_norm": 9.449095726013184, "relative_norm": 0.04999974061747816, "target_actual_norm": 9.449087142944336, "match_error": 8.58306884765625e-06, "scalar": 9.450276641371602, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.625000422965652, -0.008798462127051408], "logp": -25.633798885092702, "cache_id": 1134, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.450274467468262, "actual_norm": 9.449095726013184, "relative_norm": 0.04999974061747816, "target_actual_norm": 9.449087142944336, "match_error": 8.58306884765625e-06, "scalar": 9.450276641371602, "random_seed": 20272809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.746761496986419, "first_log_odds": -10.625, "later_log_odds": 2.8782385030135806, "first_differing_log_odds": -10.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-random1.npz", "array_sha256": "5728e76b5dd6cdf9d56652f181970a5aa56a9bc77574a06277b2a6b7342677b9", "seconds": 1.0042264591902494, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-random2", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 72176, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.44494915008545, "actual_norm": 9.44867992401123, "relative_norm": 0.04999754040775982, "target_actual_norm": 9.449087142944336, "match_error": -0.00040721893310546875, "scalar": 9.44494867966614, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1135, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Warsaw is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Warsaw is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-14.75000054196946, -2.5347081013191666], "logp": -17.284708643288624, "cache_id": 1136, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.44494915008545, "actual_norm": 9.44867992401123, "relative_norm": 0.04999754040775982, "target_actual_norm": 9.449087142944336, "match_error": -0.00040721893310546875, "scalar": 9.44494867966614, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-25.62500054196946, -0.008597292050292814], "logp": -25.633597834019753, "cache_id": 1137, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.98289489746094, "strength_one_norm": 25.49868392944336, "target_shrink_scale": 0.3705384283722583, "requested_norm": 9.44494915008545, "actual_norm": 9.44867992401123, "relative_norm": 0.04999754040775982, "target_actual_norm": 9.449087142944336, "match_error": -0.00040721893310546875, "scalar": 9.44494867966614, "random_seed": 20272810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.348889190731128, "first_log_odds": -10.875, "later_log_odds": 2.5261108092688715, "first_differing_log_odds": -10.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-random2.npz", "array_sha256": "764cd994de3682704f963527d192a47748527e16fdf92539784d6f2dcf76028d", "seconds": 0.9786449163220823, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-19-paraphrase-bridge", "case_id": "eval-19", "city": "Warsaw", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Warsaw is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Warsaw is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 72176, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 668.8009643554688, 97.03482055664062, 99.09241485595703, 16.47023582458496, 115.43370819091797, 0.36845871806144714, 81.43795013427734, 74.25067138671875, 64.85997772216797, 190.837890625, 30.28289794921875, 134.28701782226562, 47.02018356323242, 173.0908660888672, 279.8043212890625, 173.57803344726562, 98.89268493652344, 55.03424835205078, 85.36323547363281, 106.30789184570312, 24.189327239990234, 217.51390075683594, 365.3083801269531]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 668.783203125, 97.04890441894531, 99.12353515625, 16.490318298339844, 115.40914916992188, 0.22441069781780243, 81.420654296875, 74.25662994384766, 64.88878631591797, 190.83009338378906, 30.322200775146484, 134.27493286132812, 47.07236862182617, 173.0817413330078, 279.80010986328125, 173.62680053710938, 98.85533142089844, 55.099098205566406, 85.34337615966797, 106.374755859375, 24.2139892578125, 217.47158813476562, 365.3056640625]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.6142255067825317, 0.09390384703874588, 0.09059826284646988, 0.015839984640479088, 0.1060948595404625, 0.00021724832186009735, 0.07001249492168427, 0.07035432755947113, 0.060957372188568115, 0.19234929978847504, 0.03089902363717556, 0.13248088955879211, 0.04445721209049225, 0.1788000464439392, 0.3154323101043701, 0.19860117137432098, 0.0968240424990654, 0.05860505253076553, 0.07479576021432877, 0.10098724812269211, 0.02546127326786518, 0.2094820886850357, 0.38215160369873047]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1138, "cached": true, "scoring": {"normalized": "The currency used in Warsaw is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Warsaw is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-24.375160805756135, -2.3725050724222267], "logp": -26.74766587817836, "cache_id": 1139, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 668.8009643554688, 97.03482055664062, 99.09241485595703, 16.47023582458496, 115.43370819091797, 0.36845871806144714, 81.43795013427734, 74.25067138671875, 64.85997772216797, 190.837890625, 30.28289794921875, 134.28701782226562, 47.02018356323242, 173.0908660888672, 279.8043212890625, 173.57803344726562, 98.89268493652344, 55.03424835205078, 85.36323547363281, 106.30789184570312, 24.189327239990234, 217.51390075683594, 365.3083801269531]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 668.783203125, 97.04890441894531, 99.12353515625, 16.490318298339844, 115.40914916992188, 0.22441069781780243, 81.420654296875, 74.25662994384766, 64.88878631591797, 190.83009338378906, 30.322200775146484, 134.27493286132812, 47.07236862182617, 173.0817413330078, 279.80010986328125, 173.62680053710938, 98.85533142089844, 55.099098205566406, 85.34337615966797, 106.374755859375, 24.2139892578125, 217.47158813476562, 365.3056640625]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.6142255067825317, 0.09390384703874588, 0.09059826284646988, 0.015839984640479088, 0.1060948595404625, 0.00021724832186009735, 0.07001249492168427, 0.07035432755947113, 0.060957372188568115, 0.19234929978847504, 0.03089902363717556, 0.13248088955879211, 0.04445721209049225, 0.1788000464439392, 0.3154323101043701, 0.19860117137432098, 0.0968240424990654, 0.05860505253076553, 0.07479576021432877, 0.10098724812269211, 0.02546127326786518, 0.2094820886850357, 0.38215160369873047]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-13.250160805756137, -0.008825563229436423], "logp": -13.258986368985573, "cache_id": 1140, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 668.8009643554688, 97.03482055664062, 99.09241485595703, 16.47023582458496, 115.43370819091797, 0.36845871806144714, 81.43795013427734, 74.25067138671875, 64.85997772216797, 190.837890625, 30.28289794921875, 134.28701782226562, 47.02018356323242, 173.0908660888672, 279.8043212890625, 173.57803344726562, 98.89268493652344, 55.03424835205078, 85.36323547363281, 106.30789184570312, 24.189327239990234, 217.51390075683594, 365.3083801269531]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 668.783203125, 97.04890441894531, 99.12353515625, 16.490318298339844, 115.40914916992188, 0.22441069781780243, 81.420654296875, 74.25662994384766, 64.88878631591797, 190.83009338378906, 30.322200775146484, 134.27493286132812, 47.07236862182617, 173.0817413330078, 279.80010986328125, 173.62680053710938, 98.85533142089844, 55.099098205566406, 85.34337615966797, 106.374755859375, 24.2139892578125, 217.47158813476562, 365.3056640625]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.6142255067825317, 0.09390384703874588, 0.09059826284646988, 0.015839984640479088, 0.1060948595404625, 0.00021724832186009735, 0.07001249492168427, 0.07035432755947113, 0.060957372188568115, 0.19234929978847504, 0.03089902363717556, 0.13248088955879211, 0.04445721209049225, 0.1788000464439392, 0.3154323101043701, 0.19860117137432098, 0.0968240424990654, 0.05860505253076553, 0.07479576021432877, 0.10098724812269211, 0.02546127326786518, 0.2094820886850357, 0.38215160369873047]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 13.488679509192787, "first_log_odds": 11.124999999999998, "later_log_odds": 2.3636795091927887, "first_differing_log_odds": 11.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-19-paraphrase-bridge.npz", "array_sha256": "4ec8407869ae46c6462bd3a31499636b07f1e336bd5883330aca535231e604f8", "seconds": 1.2632627501152456, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-baseline", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1141, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.007217990689075742, -4.67608516620327], "logp": -4.683303156892346, "cache_id": 1142, "trace": []}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.757217990689075, -0.156344749549276], "logp": -16.91356274023835, "cache_id": 1143, "trace": []}, "log_odds": -12.230259583346005, "first_log_odds": -16.75, "later_log_odds": 4.519740416653995, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-baseline.npz", "array_sha256": "1287e9a2d57630326871392614b3b7a67298966208d56da6dc899524c7619f09", "seconds": 0.3828054592013359, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-noop", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437095642089844, "match_error": -9.437095642089844, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1144, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.007217990689075742, -4.67608516620327], "logp": -4.683303156892346, "cache_id": 1145, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437095642089844, "match_error": -9.437095642089844, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.757217990689075, -0.156344749549276], "logp": -16.91356274023835, "cache_id": 1146, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437095642089844, "match_error": -9.437095642089844, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.230259583346005, "first_log_odds": -16.75, "later_log_odds": 4.519740416653995, "first_differing_log_odds": -16.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-noop.npz", "array_sha256": "bd4f289fc694087d1b54e0da96ca8abaac5e1c444520c3b359710d83427866b9", "seconds": 0.33916037529706955, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-j", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.428887367248535, "actual_norm": 9.437095642089844, "relative_norm": 0.049998322528123924, "target_actual_norm": 9.437095642089844, "match_error": 0.0, "scalar": 0.3918435722509841, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1147, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.005894468627304368, -4.795503133189087], "logp": -4.801397601816392, "cache_id": 1148, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.428887367248535, "actual_norm": 9.437095642089844, "relative_norm": 0.049998322528123924, "target_actual_norm": 9.437095642089844, "match_error": 0.0, "scalar": 0.3918435722509841, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.880894468627304, -0.11510263320357643], "logp": -16.99599710183088, "cache_id": 1149, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.428887367248535, "actual_norm": 9.437095642089844, "relative_norm": 0.049998322528123924, "target_actual_norm": 9.437095642089844, "match_error": 0.0, "scalar": 0.3918435722509841, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.194599500014489, "first_log_odds": -16.875, "later_log_odds": 4.680400499985511, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-j.npz", "array_sha256": "289bcf2e14ed61b0314e187e90abcdb53cd187113d9c1dfd4f14ad3c8e69e795", "seconds": 0.33767416700720787, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-opposite", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.436882019042969, "actual_norm": 9.43676471710205, "relative_norm": 0.04999656926685586, "target_actual_norm": 9.437095642089844, "match_error": -0.00033092498779296875, "scalar": 9.436879644112196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1150, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.008548134073092174, -4.200667734727016], "logp": -4.209215868800109, "cache_id": 1151, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.436882019042969, "actual_norm": 9.43676471710205, "relative_norm": 0.04999656926685586, "target_actual_norm": 9.437095642089844, "match_error": -0.00033092498779296875, "scalar": 9.436879644112196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.50854813407309, -0.17264041227550098], "logp": -16.681188546348594, "cache_id": 1152, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.436882019042969, "actual_norm": 9.43676471710205, "relative_norm": 0.04999656926685586, "target_actual_norm": 9.437095642089844, "match_error": -0.00033092498779296875, "scalar": 9.436879644112196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.471972677548486, "first_log_odds": -16.5, "later_log_odds": 4.028027322451514, "first_differing_log_odds": -16.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-opposite.npz", "array_sha256": "819e4ab0a7b6941ca459a05bfa4b315c70d4f88c836130f5cd0b67521eab6a43", "seconds": 0.3398743746802211, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-unrelated", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.421977043151855, "actual_norm": 9.437005043029785, "relative_norm": 0.04999784252864132, "target_actual_norm": 9.437095642089844, "match_error": -9.059906005859375e-05, "scalar": 9.421975783654489, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1153, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.004614555736101152, -4.436156749780006], "logp": -4.440771305516106, "cache_id": 1154, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.421977043151855, "actual_norm": 9.437005043029785, "relative_norm": 0.04999784252864132, "target_actual_norm": 9.437095642089844, "match_error": -9.059906005859375e-05, "scalar": 9.421975783654489, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.8796145557361, -0.12635632680064415], "logp": -17.005970882536744, "cache_id": 1155, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.421977043151855, "actual_norm": 9.437005043029785, "relative_norm": 0.04999784252864132, "target_actual_norm": 9.437095642089844, "match_error": -9.059906005859375e-05, "scalar": 9.421975783654489, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.565199577020637, "first_log_odds": -16.875, "later_log_odds": 4.309800422979363, "first_differing_log_odds": -16.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-unrelated.npz", "array_sha256": "4dc3799e4b4751e402fd59dfc5afc2e98ba4519337687d5a067b566fbf4de5f0", "seconds": 0.34001799998804927, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-ordinary", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.435330390930176, "actual_norm": 9.437335014343262, "relative_norm": 0.049999590737283245, "target_actual_norm": 9.437095642089844, "match_error": 0.00023937225341796875, "scalar": 9.435331658605719, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1156, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.0042426232432096166, -4.672326078473814], "logp": -4.676568701717024, "cache_id": 1157, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.435330390930176, "actual_norm": 9.437335014343262, "relative_norm": 0.049999590737283245, "target_actual_norm": 9.437095642089844, "match_error": 0.00023937225341796875, "scalar": 9.435331658605719, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-17.00424262324321, -0.13975647656322598], "logp": -17.143999099806436, "cache_id": 1158, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.435330390930176, "actual_norm": 9.437335014343262, "relative_norm": 0.049999590737283245, "target_actual_norm": 9.437095642089844, "match_error": 0.00023937225341796875, "scalar": 9.435331658605719, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.467430398089412, "first_log_odds": -17.0, "later_log_odds": 4.532569601910588, "first_differing_log_odds": -17.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-ordinary.npz", "array_sha256": "7edfcd8ffc315c11ee67629480fdb23bd744fdf770f90b5922ed1bb7755543bd", "seconds": 0.34025445813313127, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-random0", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.423163414001465, "actual_norm": 9.437339782714844, "relative_norm": 0.04999961600041391, "target_actual_norm": 9.437095642089844, "match_error": 0.000244140625, "scalar": 9.423163772531552, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1159, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.0069414880578528935, -4.324131121873687], "logp": -4.33107260993154, "cache_id": 1160, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.423163414001465, "actual_norm": 9.437339782714844, "relative_norm": 0.04999961600041391, "target_actual_norm": 9.437095642089844, "match_error": 0.000244140625, "scalar": 9.423163772531552, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.63194148805785, -0.15533234387073996], "logp": -16.78727383192859, "cache_id": 1161, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.423163414001465, "actual_norm": 9.437339782714844, "relative_norm": 0.04999961600041391, "target_actual_norm": 9.437095642089844, "match_error": 0.000244140625, "scalar": 9.423163772531552, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.456201221997052, "first_log_odds": -16.625, "later_log_odds": 4.168798778002948, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-random0.npz", "array_sha256": "6cc34f3113ea2691518a3e9bacd41c926fb83a274967a0aa7697670cca1433f9", "seconds": 0.3386042919009924, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-random1", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.442710876464844, "actual_norm": 9.437087059020996, "relative_norm": 0.049998277054488734, "target_actual_norm": 9.437095642089844, "match_error": -8.58306884765625e-06, "scalar": 9.44271158950869, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1162, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.005232943448335699, -4.90900419173331], "logp": -4.914237135181645, "cache_id": 1163, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.442710876464844, "actual_norm": 9.437087059020996, "relative_norm": 0.049998277054488734, "target_actual_norm": 9.437095642089844, "match_error": -8.58306884765625e-06, "scalar": 9.44271158950869, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.630232943448334, -0.1557022140120203], "logp": -16.785935157460354, "cache_id": 1164, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.442710876464844, "actual_norm": 9.437087059020996, "relative_norm": 0.049998277054488734, "target_actual_norm": 9.437095642089844, "match_error": -8.58306884765625e-06, "scalar": 9.44271158950869, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -11.871698022278709, "first_log_odds": -16.625, "later_log_odds": 4.753301977721291, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-random1.npz", "array_sha256": "900ed25032a4e522063ec69fb483b8834627a13d6a2d6e282d8af9783932cd4a", "seconds": 0.3386127077974379, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-random2", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "PLT<|im_end|>", "output_text": "PLT", "output_ids": [2916, 51, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.443286895751953, "actual_norm": 9.436985969543457, "relative_norm": 0.049997741476118664, "target_actual_norm": 9.437095642089844, "match_error": -0.00010967254638671875, "scalar": 9.443287584115751, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1165, "cached": true, "scoring": {"normalized": "PLT", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-0.005648720593165922, -4.324032311056462], "logp": -4.329681031649629, "cache_id": 1166, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.443286895751953, "actual_norm": 9.436985969543457, "relative_norm": 0.049997741476118664, "target_actual_norm": 9.437095642089844, "match_error": -0.00010967254638671875, "scalar": 9.443287584115751, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.630648720593165, -0.12525975135279252], "logp": -16.755908471945958, "cache_id": 1167, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 188.7482452392578, "strength_one_norm": 24.06288719177246, "target_shrink_scale": 0.3918435722509841, "requested_norm": 9.443286895751953, "actual_norm": 9.436985969543457, "relative_norm": 0.049997741476118664, "target_actual_norm": 9.437095642089844, "match_error": -0.00010967254638671875, "scalar": 9.443287584115751, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.42622744029633, "first_log_odds": -16.625, "later_log_odds": 4.19877255970367, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-random2.npz", "array_sha256": "6befde63c78a9aba1861e9da32e69b8ce5178efbd94ce8c833ec729caf1546d5", "seconds": 0.34230066696181893, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-main-bridge", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 88.0865249633789, 388.9857482910156, 85.1334228515625, 183.377685546875, 69.28980255126953, 39.41938018798828, 11.619632720947266, 82.64378356933594, 107.33672332763672, 67.70279693603516, 182.07911682128906, 15.761287689208984, 133.60118103027344, 67.07168579101562, 57.68590545654297, 90.6223373413086, 122.55575561523438, 97.47116088867188, 319.6104736328125, 147.66665649414062, 105.16741180419922, 93.42984771728516, 84.49027252197266, 124.3896713256836, 4.755014896392822, 217.00523376464844, 437.18499755859375]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 88.07784271240234, 389.0208435058594, 85.1639404296875, 183.3819122314453, 69.28435516357422, 39.463016510009766, 11.70872688293457, 82.65248107910156, 107.40438842773438, 67.64318084716797, 182.15277099609375, 15.801132202148438, 133.56002807617188, 67.09121704101562, 57.669246673583984, 90.62744903564453, 122.56344604492188, 97.46830749511719, 319.65081787109375, 147.65411376953125, 105.2095718383789, 93.48766326904297, 84.4735336303711, 124.4303207397461, 4.936145305633545, 216.99388122558594, 437.2232971191406]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.06916822493076324, 0.3890310227870941, 0.08903466910123825, 0.17715491354465485, 0.06466200947761536, 0.038068339228630066, 0.01137145608663559, 0.07261772453784943, 0.1054290384054184, 0.06474551558494568, 0.18550987541675568, 0.016391394659876823, 0.13451321423053741, 0.06412222236394882, 0.05514439940452576, 0.07254376262426376, 0.12255541235208511, 0.1089225634932518, 0.35344570875167847, 0.17876911163330078, 0.1047130897641182, 0.09864427149295807, 0.07501634955406189, 0.12121190130710602, 0.005308239720761776, 0.20970772206783295, 0.40469416975975037]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1168, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-9.26288033708103, -4.367571944638716], "logp": -13.630452281719746, "cache_id": 1169, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 88.0865249633789, 388.9857482910156, 85.1334228515625, 183.377685546875, 69.28980255126953, 39.41938018798828, 11.619632720947266, 82.64378356933594, 107.33672332763672, 67.70279693603516, 182.07911682128906, 15.761287689208984, 133.60118103027344, 67.07168579101562, 57.68590545654297, 90.6223373413086, 122.55575561523438, 97.47116088867188, 319.6104736328125, 147.66665649414062, 105.16741180419922, 93.42984771728516, 84.49027252197266, 124.3896713256836, 4.755014896392822, 217.00523376464844, 437.18499755859375]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 88.07784271240234, 389.0208435058594, 85.1639404296875, 183.3819122314453, 69.28435516357422, 39.463016510009766, 11.70872688293457, 82.65248107910156, 107.40438842773438, 67.64318084716797, 182.15277099609375, 15.801132202148438, 133.56002807617188, 67.09121704101562, 57.669246673583984, 90.62744903564453, 122.56344604492188, 97.46830749511719, 319.65081787109375, 147.65411376953125, 105.2095718383789, 93.48766326904297, 84.4735336303711, 124.4303207397461, 4.936145305633545, 216.99388122558594, 437.2232971191406]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.06916822493076324, 0.3890310227870941, 0.08903466910123825, 0.17715491354465485, 0.06466200947761536, 0.038068339228630066, 0.01137145608663559, 0.07261772453784943, 0.1054290384054184, 0.06474551558494568, 0.18550987541675568, 0.016391394659876823, 0.13451321423053741, 0.06412222236394882, 0.05514439940452576, 0.07254376262426376, 0.12255541235208511, 0.1089225634932518, 0.35344570875167847, 0.17876911163330078, 0.1047130897641182, 0.09864427149295807, 0.07501634955406189, 0.12121190130710602, 0.005308239720761776, 0.20970772206783295, 0.40469416975975037]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.1378803370810298, -0.06842271772197976], "logp": -0.2063030548030096, "cache_id": 1170, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 92.99726867675781, 202.03829956054688, 43.05839920043945, 41.82255172729492, 170.83682250976562, 88.0865249633789, 388.9857482910156, 85.1334228515625, 183.377685546875, 69.28980255126953, 39.41938018798828, 11.619632720947266, 82.64378356933594, 107.33672332763672, 67.70279693603516, 182.07911682128906, 15.761287689208984, 133.60118103027344, 67.07168579101562, 57.68590545654297, 90.6223373413086, 122.55575561523438, 97.47116088867188, 319.6104736328125, 147.66665649414062, 105.16741180419922, 93.42984771728516, 84.49027252197266, 124.3896713256836, 4.755014896392822, 217.00523376464844, 437.18499755859375]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 93.00401306152344, 202.09034729003906, 43.09341049194336, 41.884742736816406, 170.8494415283203, 88.07784271240234, 389.0208435058594, 85.1639404296875, 183.3819122314453, 69.28435516357422, 39.463016510009766, 11.70872688293457, 82.65248107910156, 107.40438842773438, 67.64318084716797, 182.15277099609375, 15.801132202148438, 133.56002807617188, 67.09121704101562, 57.669246673583984, 90.62744903564453, 122.56344604492188, 97.46830749511719, 319.65081787109375, 147.65411376953125, 105.2095718383789, 93.48766326904297, 84.4735336303711, 124.4303207397461, 4.936145305633545, 216.99388122558594, 437.2232971191406]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.08537441492080688, 0.1773103028535843, 0.035058774054050446, 0.03584999963641167, 0.1505526602268219, 0.06916822493076324, 0.3890310227870941, 0.08903466910123825, 0.17715491354465485, 0.06466200947761536, 0.038068339228630066, 0.01137145608663559, 0.07261772453784943, 0.1054290384054184, 0.06474551558494568, 0.18550987541675568, 0.016391394659876823, 0.13451321423053741, 0.06412222236394882, 0.05514439940452576, 0.07254376262426376, 0.12255541235208511, 0.1089225634932518, 0.35344570875167847, 0.17876911163330078, 0.1047130897641182, 0.09864427149295807, 0.07501634955406189, 0.12121190130710602, 0.005308239720761776, 0.20970772206783295, 0.40469416975975037]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 13.424149226916736, "first_log_odds": 9.125, "later_log_odds": 4.299149226916736, "first_differing_log_odds": 9.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-main-bridge.npz", "array_sha256": "8ea474654a6c065c705a3fa0caf1ca671aa5d9c0ef5db549ef240b0d51a20f57", "seconds": 0.3436307921074331, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-baseline", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1171, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-10.750023707030087, -5.157170142184358], "logp": -15.907193849214444, "cache_id": 1172, "trace": []}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.625023707030085, -0.023558975929679142], "logp": -23.648582682959763, "cache_id": 1173, "trace": []}, "log_odds": -7.741388833745319, "first_log_odds": -12.874999999999998, "later_log_odds": 5.1336111662546795, "first_differing_log_odds": -12.874999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-baseline.npz", "array_sha256": "d54b540fe16ea982e41e44b5dc94c9f09e065675d1dd2d14abed4e0e021642e0", "seconds": 1.3040944579988718, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-noop", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.435098648071289, "match_error": -9.435098648071289, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1174, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-10.750023707030087, -5.157170142184358], "logp": -15.907193849214444, "cache_id": 1175, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.435098648071289, "match_error": -9.435098648071289, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.625023707030085, -0.023558975929679142], "logp": -23.648582682959763, "cache_id": 1176, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.435098648071289, "match_error": -9.435098648071289, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.741388833745319, "first_log_odds": -12.874999999999998, "later_log_odds": 5.1336111662546795, "first_differing_log_odds": -12.874999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-noop.npz", "array_sha256": "938a1724ec2f4bde586909fa39166f69fa48d5972a25a17f6986ab481c188338", "seconds": 1.2892394997179508, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-j", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.431127548217773, "actual_norm": 9.435098648071289, "relative_norm": 0.049999411737022915, "target_actual_norm": 9.435098648071289, "match_error": 0.0, "scalar": 0.4292956142928822, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1177, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-11.37501327498103, -5.526560864768425], "logp": -16.901574139749457, "cache_id": 1178, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.431127548217773, "actual_norm": 9.435098648071289, "relative_norm": 0.049999411737022915, "target_actual_norm": 9.435098648071289, "match_error": 0.0, "scalar": 0.4292956142928822, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.50001327498103, -0.019447907916471215], "logp": -23.5194611828975, "cache_id": 1179, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.431127548217773, "actual_norm": 9.435098648071289, "relative_norm": 0.049999411737022915, "target_actual_norm": 9.435098648071289, "match_error": 0.0, "scalar": 0.4292956142928822, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.617887043148045, "first_log_odds": -12.124999999999998, "later_log_odds": 5.5071129568519535, "first_differing_log_odds": -12.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-j.npz", "array_sha256": "ded8e55335729620e51037e374613422c01fb2d0f6e3df3518b8cb7ae97c569d", "seconds": 1.2863615001551807, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-opposite", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436899185180664, "actual_norm": 9.434117317199707, "relative_norm": 0.04999421137101418, "target_actual_norm": 9.435098648071289, "match_error": -0.0009813308715820312, "scalar": 9.436898250321974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1180, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-10.500030721806587, -4.58578453678253], "logp": -15.085815258589117, "cache_id": 1181, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436899185180664, "actual_norm": 9.434117317199707, "relative_norm": 0.04999421137101418, "target_actual_norm": 9.435098648071289, "match_error": -0.0009813308715820312, "scalar": 9.436898250321974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.62503072180659, -0.023367337469079767], "logp": -23.648398059275667, "cache_id": 1182, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436899185180664, "actual_norm": 9.434117317199707, "relative_norm": 0.04999421137101418, "target_actual_norm": 9.435098648071289, "match_error": -0.0009813308715820312, "scalar": 9.436898250321974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.56258280068655, "first_log_odds": -13.125000000000002, "later_log_odds": 4.562417199313451, "first_differing_log_odds": -13.125000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-opposite.npz", "array_sha256": "eb6182eca14d4df9f39c13fae389d55816724aa82c3bfff602cbdc41d434812c", "seconds": 1.2913082079030573, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-unrelated", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.429017066955566, "actual_norm": 9.435087203979492, "relative_norm": 0.04999935109135518, "target_actual_norm": 9.435098648071289, "match_error": -1.1444091796875e-05, "scalar": 9.429015992463974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1183, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-11.000019164828034, -5.278465563749254], "logp": -16.27848472857729, "cache_id": 1184, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.429017066955566, "actual_norm": 9.435087203979492, "relative_norm": 0.04999935109135518, "target_actual_norm": 9.435098648071289, "match_error": -1.1444091796875e-05, "scalar": 9.429015992463974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.875019164828036, -0.01895665109040548], "logp": -23.89397581591844, "cache_id": 1185, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.429017066955566, "actual_norm": 9.435087203979492, "relative_norm": 0.04999935109135518, "target_actual_norm": 9.435098648071289, "match_error": -1.1444091796875e-05, "scalar": 9.429015992463974, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.615491087341152, "first_log_odds": -12.875000000000002, "later_log_odds": 5.25950891265885, "first_differing_log_odds": -12.875000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-unrelated.npz", "array_sha256": "ddd8084d35ee9e8b42bfa01e217bcdea7273df18b3a5cdaa8a6272f21c920c77", "seconds": 1.2900005830451846, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-ordinary", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.435277938842773, "actual_norm": 9.434982299804688, "relative_norm": 0.04999879517273423, "target_actual_norm": 9.435098648071289, "match_error": -0.0001163482666015625, "scalar": 9.435278608296358, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1186, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-11.000019004994305, -5.156775336366916], "logp": -16.15679434136122, "cache_id": 1187, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.435277938842773, "actual_norm": 9.434982299804688, "relative_norm": 0.04999879517273423, "target_actual_norm": 9.435098648071289, "match_error": -0.0001163482666015625, "scalar": 9.435278608296358, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.625019004994307, -0.02100952327369899], "logp": -23.646028528268005, "cache_id": 1188, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.435277938842773, "actual_norm": 9.434982299804688, "relative_norm": 0.04999879517273423, "target_actual_norm": 9.435098648071289, "match_error": -0.0001163482666015625, "scalar": 9.435278608296358, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.489234186906785, "first_log_odds": -12.625000000000002, "later_log_odds": 5.135765813093217, "first_differing_log_odds": -12.625000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-ordinary.npz", "array_sha256": "dcc55b4fa0d3988dfdfad2d73240f6e351d0f32f956dc795be86396685a801be", "seconds": 1.3043006667867303, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-random0", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.42358112335205, "actual_norm": 9.435101509094238, "relative_norm": 0.04999942689843986, "target_actual_norm": 9.435098648071289, "match_error": 2.86102294921875e-06, "scalar": 9.423581193666905, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1189, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-10.75002472752116, -5.4086981396838], "logp": -16.158722867204958, "cache_id": 1190, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.42358112335205, "actual_norm": 9.435101509094238, "relative_norm": 0.04999942689843986, "target_actual_norm": 9.435098648071289, "match_error": 2.86102294921875e-06, "scalar": 9.423581193666905, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.62502472752116, -0.02160485029464631], "logp": -23.64662957781581, "cache_id": 1191, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.42358112335205, "actual_norm": 9.435101509094238, "relative_norm": 0.04999942689843986, "target_actual_norm": 9.435098648071289, "match_error": 2.86102294921875e-06, "scalar": 9.423581193666905, "random_seed": 20272908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.487906710610851, "first_log_odds": -12.875000000000002, "later_log_odds": 5.387093289389151, "first_differing_log_odds": -12.875000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-random0.npz", "array_sha256": "275be6983172279d89fa5c1074e8208d2bdc95eb5e39d4eb628a841c26647153", "seconds": 1.2848440832458436, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-random1", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436034202575684, "actual_norm": 9.435112953186035, "relative_norm": 0.049999487544107594, "target_actual_norm": 9.435098648071289, "match_error": 1.430511474609375e-05, "scalar": 9.436034441241645, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1192, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-10.500030063369934, -5.154414030186196], "logp": -15.65444409355613, "cache_id": 1193, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436034202575684, "actual_norm": 9.435112953186035, "relative_norm": 0.049999487544107594, "target_actual_norm": 9.435098648071289, "match_error": 1.430511474609375e-05, "scalar": 9.436034441241645, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.500030063369934, -0.025786182495785164], "logp": -23.52581624586572, "cache_id": 1194, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.436034202575684, "actual_norm": 9.435112953186035, "relative_norm": 0.049999487544107594, "target_actual_norm": 9.435098648071289, "match_error": 1.430511474609375e-05, "scalar": 9.436034441241645, "random_seed": 20272909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.87137215230959, "first_log_odds": -13.0, "later_log_odds": 5.12862784769041, "first_differing_log_odds": -13.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-random1.npz", "array_sha256": "e2a70d33cf0cb96ba77699d0b00fcb6339ebeb55f3551e25fdf02e555522a619", "seconds": 1.3227388341911137, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-random2", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.442980766296387, "actual_norm": 9.435162544250488, "relative_norm": 0.04999975034200114, "target_actual_norm": 9.435098648071289, "match_error": 6.389617919921875e-05, "scalar": 9.44298090592929, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1195, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-11.00001953944807, -5.040196571495426], "logp": -16.040216110943497, "cache_id": 1196, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.442980766296387, "actual_norm": 9.435162544250488, "relative_norm": 0.04999975034200114, "target_actual_norm": 9.435098648071289, "match_error": 6.389617919921875e-05, "scalar": 9.44298090592929, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-23.500019539448072, -0.019832085393906962], "logp": -23.51985162484198, "cache_id": 1197, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 188.70419311523438, "strength_one_norm": 21.968843460083008, "target_shrink_scale": 0.4292956142928822, "requested_norm": 9.442980766296387, "actual_norm": 9.435162544250488, "relative_norm": 0.04999975034200114, "target_actual_norm": 9.435098648071289, "match_error": 6.389617919921875e-05, "scalar": 9.44298090592929, "random_seed": 20272910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.479635513898483, "first_log_odds": -12.500000000000002, "later_log_odds": 5.020364486101519, "first_differing_log_odds": -12.500000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-random2.npz", "array_sha256": "132ab7affec3a87ef30e99fb07bca6e09b29e822d5e2d204aae5d010523ee3b5", "seconds": 1.2914348752237856, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-20-paraphrase-bridge", "case_id": "eval-20", "city": "Poznan", "country": "Poland", "code": "PLN", "target_country": "Hungary", "target_code": "HUF", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.<|im_end|>", "output_text": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "output_ids": [785, 11413, 1483, 304, 78985, 18759, 374, 279, 31984, 1863, 4594, 354, 88, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2916, 45, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 115.17050170898438, 435.23736572265625, 66.81008911132812, 114.35633087158203, 47.0275993347168, 121.92108154296875, 10.73650074005127, 72.93053436279297, 80.08613586425781, 51.66171646118164, 190.1066131591797, 33.84060287475586, 129.56427001953125, 30.97508430480957, 108.80776977539062, 280.89837646484375, 164.16793823242188, 99.28833770751953, 55.782806396484375, 68.90962219238281, 113.07572937011719, 13.261606216430664, 224.8509521484375, 364.8925476074219]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 115.31385803222656, 435.1990661621094, 66.84578704833984, 114.3699722290039, 46.994232177734375, 121.9078369140625, 10.783219337463379, 72.96116638183594, 80.07646942138672, 51.730133056640625, 190.1273956298828, 33.86068344116211, 129.51731872558594, 31.003881454467773, 108.79044342041016, 280.88818359375, 164.15565490722656, 99.29268646240234, 55.809844970703125, 69.00153350830078, 113.09286499023438, 13.318119049072266, 224.79054260253906, 364.8438415527344]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.08968263119459152, 0.43118950724601746, 0.0685620978474617, 0.10542478412389755, 0.04542562738060951, 0.11255136132240295, 0.01049538142979145, 0.06258753687143326, 0.07616785168647766, 0.04852336272597313, 0.1924518346786499, 0.034735821187496185, 0.12728647887706757, 0.029394453391432762, 0.11520761251449585, 0.3173266053199768, 0.19081814587116241, 0.09713476151227951, 0.05915454775094986, 0.06078367307782173, 0.10734570026397705, 0.014028424397110939, 0.2157953828573227, 0.37888866662979126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1198, "cached": true, "scoring": {"normalized": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **PLN**.", "bare_format": false, "malformed": true, "first_code": "PLN", "all_codes": ["PLN"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Poznan is the Polish Złoty, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Polish", "start": 35, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2916, 45], "tokens": ["PL", "N"], "log_probs": [-20.876522823215854, -5.036186840124293], "logp": -25.912709663340145, "cache_id": 1199, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 115.17050170898438, 435.23736572265625, 66.81008911132812, 114.35633087158203, 47.0275993347168, 121.92108154296875, 10.73650074005127, 72.93053436279297, 80.08613586425781, 51.66171646118164, 190.1066131591797, 33.84060287475586, 129.56427001953125, 30.97508430480957, 108.80776977539062, 280.89837646484375, 164.16793823242188, 99.28833770751953, 55.782806396484375, 68.90962219238281, 113.07572937011719, 13.261606216430664, 224.8509521484375, 364.8925476074219]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 115.31385803222656, 435.1990661621094, 66.84578704833984, 114.3699722290039, 46.994232177734375, 121.9078369140625, 10.783219337463379, 72.96116638183594, 80.07646942138672, 51.730133056640625, 190.1273956298828, 33.86068344116211, 129.51731872558594, 31.003881454467773, 108.79044342041016, 280.88818359375, 164.15565490722656, 99.29268646240234, 55.809844970703125, 69.00153350830078, 113.09286499023438, 13.318119049072266, 224.79054260253906, 364.8438415527344]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.08968263119459152, 0.43118950724601746, 0.0685620978474617, 0.10542478412389755, 0.04542562738060951, 0.11255136132240295, 0.01049538142979145, 0.06258753687143326, 0.07616785168647766, 0.04852336272597313, 0.1924518346786499, 0.034735821187496185, 0.12728647887706757, 0.029394453391432762, 0.11520761251449585, 0.3173266053199768, 0.19081814587116241, 0.09713476151227951, 0.05915454775094986, 0.06078367307782173, 0.10734570026397705, 0.014028424397110939, 0.2157953828573227, 0.37888866662979126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-10.876522823215852, -0.01866668367333414], "logp": -10.895189506889187, "cache_id": 1200, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[39.39202117919922, 180.06585693359375, 47.644405364990234, 20.200613021850586, 115.956298828125, 171.07196044921875, 88.15290069580078, 167.76608276367188, 115.17050170898438, 435.23736572265625, 66.81008911132812, 114.35633087158203, 47.0275993347168, 121.92108154296875, 10.73650074005127, 72.93053436279297, 80.08613586425781, 51.66171646118164, 190.1066131591797, 33.84060287475586, 129.56427001953125, 30.97508430480957, 108.80776977539062, 280.89837646484375, 164.16793823242188, 99.28833770751953, 55.782806396484375, 68.90962219238281, 113.07572937011719, 13.261606216430664, 224.8509521484375, 364.8925476074219]], "actual_norm": [[39.43375015258789, 180.02455139160156, 47.66461181640625, 20.24420928955078, 115.97773742675781, 171.02188110351562, 88.13372039794922, 167.784423828125, 115.31385803222656, 435.1990661621094, 66.84578704833984, 114.3699722290039, 46.994232177734375, 121.9078369140625, 10.783219337463379, 72.96116638183594, 80.07646942138672, 51.730133056640625, 190.1273956298828, 33.86068344116211, 129.51731872558594, 31.003881454467773, 108.79044342041016, 280.88818359375, 164.15565490722656, 99.29268646240234, 55.809844970703125, 69.00153350830078, 113.09286499023438, 13.318119049072266, 224.79054260253906, 364.8438415527344]], "relative_norm": [[0.002301545813679695, 0.19096621870994568, 0.052731920033693314, 0.01765083335340023, 0.10557165741920471, 0.14857284724712372, 0.0783851370215416, 0.1490439474582672, 0.08968263119459152, 0.43118950724601746, 0.0685620978474617, 0.10542478412389755, 0.04542562738060951, 0.11255136132240295, 0.01049538142979145, 0.06258753687143326, 0.07616785168647766, 0.04852336272597313, 0.1924518346786499, 0.034735821187496185, 0.12728647887706757, 0.029394453391432762, 0.11520761251449585, 0.3173266053199768, 0.19081814587116241, 0.09713476151227951, 0.05915454775094986, 0.06078367307782173, 0.10734570026397705, 0.014028424397110939, 0.2157953828573227, 0.37888866662979126]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.017520156450958, "first_log_odds": 10.000000000000002, "later_log_odds": 5.017520156450956, "first_differing_log_odds": 10.000000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-20-paraphrase-bridge.npz", "array_sha256": "bb5a2f5b89c05203b1d0909280a0cb9412defc803783f8125ff38b38afb5c86a", "seconds": 1.2926973751746118, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-baseline", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1201, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-2.2751051739047074e-05, -0.00258327728545168], "logp": -0.002606028337190727, "cache_id": 1202, "trace": []}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.56252275105174, -0.6193466115758234], "logp": -33.18186936262756, "cache_id": 1203, "trace": []}, "log_odds": -33.17926333429037, "first_log_odds": -32.5625, "later_log_odds": -0.6167633342903684, "first_differing_log_odds": -32.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-baseline.npz", "array_sha256": "7147b4440c79b1e0adc13719e5c00abeae6506f0189cc133cb887eecef891065", "seconds": 0.39214720856398344, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457370624, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-noop", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.4130277633667, "match_error": -9.4130277633667, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1204, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-2.2751051739047074e-05, -0.00258327728545168], "logp": -0.002606028337190727, "cache_id": 1205, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.4130277633667, "match_error": -9.4130277633667, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.56252275105174, -0.6193466115758234], "logp": -33.18186936262756, "cache_id": 1206, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.4130277633667, "match_error": -9.4130277633667, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -33.17926333429037, "first_log_odds": -32.5625, "later_log_odds": -0.6167633342903684, "first_differing_log_odds": -32.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-noop.npz", "array_sha256": "647bb1d62519c25fee75a094ad5ab371ec01bde95816b3dbfed03ba0d8bf872d", "seconds": 0.3421620409935713, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457403392, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-j", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.425559043884277, "actual_norm": 9.4130277633667, "relative_norm": 0.04993352399077598, "target_actual_norm": 9.4130277633667, "match_error": 0.0, "scalar": 0.13978355405468706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1207, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-2.687369324649646e-05, -0.003092290719611274], "logp": -0.0031191644128577703, "cache_id": 1208, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.425559043884277, "actual_norm": 9.4130277633667, "relative_norm": 0.04993352399077598, "target_actual_norm": 9.4130277633667, "match_error": 0.0, "scalar": 0.13978355405468706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-31.187526873693248, -0.26356382085291397], "logp": -31.45109069454616, "cache_id": 1209, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.425559043884277, "actual_norm": 9.4130277633667, "relative_norm": 0.04993352399077598, "target_actual_norm": 9.4130277633667, "match_error": 0.0, "scalar": 0.13978355405468706, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -31.4479715301333, "first_log_odds": -31.1875, "later_log_odds": -0.2604715301333016, "first_differing_log_odds": -31.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-j.npz", "array_sha256": "e592b2d0b4fe7e55a170bad81c2a24e0694a38902030abde7a505cf7a49ecc3d", "seconds": 0.339454541914165, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457403392, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-opposite", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.421287536621094, "actual_norm": 9.413090705871582, "relative_norm": 0.049933857883457125, "target_actual_norm": 9.4130277633667, "match_error": 6.29425048828125e-05, "scalar": 9.421286569158838, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1210, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-2.3199476581044827e-05, -0.00202281858370247], "logp": -0.002046018060283515, "cache_id": 1211, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.421287536621094, "actual_norm": 9.413090705871582, "relative_norm": 0.049933857883457125, "target_actual_norm": 9.4130277633667, "match_error": 6.29425048828125e-05, "scalar": 9.421286569158838, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-33.25002319947658, -0.7813939938427], "logp": -34.03141719331928, "cache_id": 1212, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.421287536621094, "actual_norm": 9.413090705871582, "relative_norm": 0.049933857883457125, "target_actual_norm": 9.4130277633667, "match_error": 6.29425048828125e-05, "scalar": 9.421286569158838, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -34.029371175258994, "first_log_odds": -33.25, "later_log_odds": -0.7793711752589942, "first_differing_log_odds": -33.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-opposite.npz", "array_sha256": "d872a56cd98295339c751f6e777f69a407a04a9a8f6b46142bcef44b0db10a95", "seconds": 0.3428656249307096, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457403392, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-unrelated", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416061401367188, "actual_norm": 9.413206100463867, "relative_norm": 0.04993447002003923, "target_actual_norm": 9.4130277633667, "match_error": 0.00017833709716796875, "scalar": 9.416061976799028, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1213, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-3.0100013829305363e-05, -0.002107270685011564], "logp": -0.0021373706988408693, "cache_id": 1214, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416061401367188, "actual_norm": 9.413206100463867, "relative_norm": 0.04993447002003923, "target_actual_norm": 9.4130277633667, "match_error": 0.00017833709716796875, "scalar": 9.416061976799028, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.43753010001383, -0.49599123806775025], "logp": -32.93352133808158, "cache_id": 1215, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416061401367188, "actual_norm": 9.413206100463867, "relative_norm": 0.04993447002003923, "target_actual_norm": 9.4130277633667, "match_error": 0.00017833709716796875, "scalar": 9.416061976799028, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -32.93138396738274, "first_log_odds": -32.4375, "later_log_odds": -0.4938839673827431, "first_differing_log_odds": -32.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-unrelated.npz", "array_sha256": "4825726b1c36a280845a3ed95827bd650d603ec9037765379cc33744d49ae6f4", "seconds": 0.34050024999305606, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457403392, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-ordinary", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402129173278809, "actual_norm": 9.413019180297852, "relative_norm": 0.04993347845995583, "target_actual_norm": 9.4130277633667, "match_error": -8.58306884765625e-06, "scalar": 9.402129730506203, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1216, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-1.9219768108808383e-05, -0.0021911434161123356], "logp": -0.002210363184221144, "cache_id": 1217, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402129173278809, "actual_norm": 9.413019180297852, "relative_norm": 0.04993347845995583, "target_actual_norm": 9.4130277633667, "match_error": -8.58306884765625e-06, "scalar": 9.402129730506203, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.312519219768106, -0.5001954671499211], "logp": -32.81271468691803, "cache_id": 1218, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402129173278809, "actual_norm": 9.413019180297852, "relative_norm": 0.04993347845995583, "target_actual_norm": 9.4130277633667, "match_error": -8.58306884765625e-06, "scalar": 9.402129730506203, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -32.81050432373381, "first_log_odds": -32.3125, "later_log_odds": -0.49800432373380943, "first_differing_log_odds": -32.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-ordinary.npz", "array_sha256": "8984b0f7d686dd7f754715630284e56e20a737a7ec2c148205c69530969eeb26", "seconds": 0.34351012483239174, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457436160, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-random0", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.40699291229248, "actual_norm": 9.413086891174316, "relative_norm": 0.04993383764753706, "target_actual_norm": 9.4130277633667, "match_error": 5.91278076171875e-05, "scalar": 9.406995244353311, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1219, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-1.9116509722728992e-05, -0.0025497951581835185], "logp": -0.0025689116679062473, "cache_id": 1220, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.40699291229248, "actual_norm": 9.413086891174316, "relative_norm": 0.04993383764753706, "target_actual_norm": 9.4130277633667, "match_error": 5.91278076171875e-05, "scalar": 9.406995244353311, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.56251911650972, -0.7736412429554261], "logp": -33.33616035946515, "cache_id": 1221, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.40699291229248, "actual_norm": 9.413086891174316, "relative_norm": 0.04993383764753706, "target_actual_norm": 9.4130277633667, "match_error": 5.91278076171875e-05, "scalar": 9.406995244353311, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -33.33359144779724, "first_log_odds": -32.5625, "later_log_odds": -0.7710914477972395, "first_differing_log_odds": -32.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-random0.npz", "array_sha256": "4fd48060c125df04695ce36e2ff0321fa2cbf421904241e85f788698cbdbda38", "seconds": 0.3402563747949898, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457436160, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-random1", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416302680969238, "actual_norm": 9.412970542907715, "relative_norm": 0.049933220451974944, "target_actual_norm": 9.4130277633667, "match_error": -5.7220458984375e-05, "scalar": 9.416304354795102, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1222, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-3.0121094681984267e-05, -0.0027091190589931405], "logp": -0.0027392401536751247, "cache_id": 1223, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416302680969238, "actual_norm": 9.412970542907715, "relative_norm": 0.049933220451974944, "target_actual_norm": 9.4130277633667, "match_error": -5.7220458984375e-05, "scalar": 9.416304354795102, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-31.81253012109468, -0.39419828169981913], "logp": -32.2067284027945, "cache_id": 1224, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.416302680969238, "actual_norm": 9.412970542907715, "relative_norm": 0.049933220451974944, "target_actual_norm": 9.4130277633667, "match_error": -5.7220458984375e-05, "scalar": 9.416304354795102, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -32.20398916264083, "first_log_odds": -31.8125, "later_log_odds": -0.39148916264083056, "first_differing_log_odds": -31.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-random1.npz", "array_sha256": "130fbcdec33a05d755f16eb6b28a5733d7d9ae371d78c2d638345ff105eedf48", "seconds": 0.34058108320459723, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457436160, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-random2", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402595520019531, "actual_norm": 9.413055419921875, "relative_norm": 0.04993367070119648, "target_actual_norm": 9.4130277633667, "match_error": 2.765655517578125e-05, "scalar": 9.402596532572716, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1225, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-2.343368881230429e-05, -0.002431709812482347], "logp": -0.002455143501294651, "cache_id": 1226, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402595520019531, "actual_norm": 9.413055419921875, "relative_norm": 0.04993367070119648, "target_actual_norm": 9.4130277633667, "match_error": 2.765655517578125e-05, "scalar": 9.402596532572716, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-32.43752343368881, -0.42218780555638313], "logp": -32.85971123924519, "cache_id": 1227, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 188.5111846923828, "strength_one_norm": 67.42967224121094, "target_shrink_scale": 0.13978355405468706, "requested_norm": 9.402595520019531, "actual_norm": 9.413055419921875, "relative_norm": 0.04993367070119648, "target_actual_norm": 9.4130277633667, "match_error": 2.765655517578125e-05, "scalar": 9.402596532572716, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -32.8572560957439, "first_log_odds": -32.4375, "later_log_odds": -0.4197560957438995, "first_differing_log_odds": -32.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-random2.npz", "array_sha256": "7386d4bbe84021ad0fb3906fbee137b2dc354bf8508e0f9db7504f12dd21ba5a", "seconds": 0.33965454204007983, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457436160, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-main-bridge", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 920.190185546875, 332.6709899902344, 152.66966247558594, 254.60581970214844, 182.3895263671875, 262.418701171875, 254.6576690673828, 232.83184814453125, 189.8848876953125, 191.2544708251953, 115.37710571289062, 247.80101013183594, 248.46263122558594, 120.34003448486328, 207.76693725585938, 37.36530685424805, 327.0835266113281, 176.74063110351562, 147.10153198242188, 102.71269226074219, 34.77437210083008, 209.5, 200.78515625, 216.61680603027344, 1.3171651363372803, 1193.21044921875]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 920.3029174804688, 332.64593505859375, 152.67721557617188, 254.61549377441406, 182.37371826171875, 262.41015625, 254.66915893554688, 232.82122802734375, 189.87265014648438, 191.2648162841797, 115.397216796875, 247.774169921875, 248.4126434326172, 120.35723876953125, 207.77537536621094, 37.34035110473633, 327.09368896484375, 176.76461791992188, 147.11834716796875, 102.72297668457031, 34.854373931884766, 209.44004821777344, 200.7854766845703, 216.6543731689453, 1.23737370967865, 1193.26904296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.8704268932342529, 0.3473832309246063, 0.1495087593793869, 0.23812329769134521, 0.17850947380065918, 0.254533976316452, 0.22376105189323425, 0.2285299003124237, 0.18161724507808685, 0.19555462896823883, 0.11946316063404083, 0.24775850772857666, 0.23793382942676544, 0.11462461203336716, 0.1670796126127243, 0.03747349604964256, 0.3640751838684082, 0.19524036347866058, 0.17405372858047485, 0.10189177095890045, 0.03714651241898537, 0.18566389381885529, 0.19320346415042877, 0.23296740651130676, 0.0011953806970268488, 1.046237826347351]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}], "cache_id": 1228, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-22.456847345599307, -0.0010258450794158212], "logp": -22.457873190678722, "cache_id": 1229, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 920.190185546875, 332.6709899902344, 152.66966247558594, 254.60581970214844, 182.3895263671875, 262.418701171875, 254.6576690673828, 232.83184814453125, 189.8848876953125, 191.2544708251953, 115.37710571289062, 247.80101013183594, 248.46263122558594, 120.34003448486328, 207.76693725585938, 37.36530685424805, 327.0835266113281, 176.74063110351562, 147.10153198242188, 102.71269226074219, 34.77437210083008, 209.5, 200.78515625, 216.61680603027344, 1.3171651363372803, 1193.21044921875]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 920.3029174804688, 332.64593505859375, 152.67721557617188, 254.61549377441406, 182.37371826171875, 262.41015625, 254.66915893554688, 232.82122802734375, 189.87265014648438, 191.2648162841797, 115.397216796875, 247.774169921875, 248.4126434326172, 120.35723876953125, 207.77537536621094, 37.34035110473633, 327.09368896484375, 176.76461791992188, 147.11834716796875, 102.72297668457031, 34.854373931884766, 209.44004821777344, 200.7854766845703, 216.6543731689453, 1.23737370967865, 1193.26904296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.8704268932342529, 0.3473832309246063, 0.1495087593793869, 0.23812329769134521, 0.17850947380065918, 0.254533976316452, 0.22376105189323425, 0.2285299003124237, 0.18161724507808685, 0.19555462896823883, 0.11946316063404083, 0.24775850772857666, 0.23793382942676544, 0.11462461203336716, 0.1670796126127243, 0.03747349604964256, 0.3640751838684082, 0.19524036347866058, 0.17405372858047485, 0.10189177095890045, 0.03714651241898537, 0.18566389381885529, 0.19320346415042877, 0.23296740651130676, 0.0011953806970268488, 1.046237826347351]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.081847345599309, -0.5139778047740257], "logp": -7.595825150373335, "cache_id": 1230, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 920.190185546875, 332.6709899902344, 152.66966247558594, 254.60581970214844, 182.3895263671875, 262.418701171875, 254.6576690673828, 232.83184814453125, 189.8848876953125, 191.2544708251953, 115.37710571289062, 247.80101013183594, 248.46263122558594, 120.34003448486328, 207.76693725585938, 37.36530685424805, 327.0835266113281, 176.74063110351562, 147.10153198242188, 102.71269226074219, 34.77437210083008, 209.5, 200.78515625, 216.61680603027344, 1.3171651363372803, 1193.21044921875]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 920.3029174804688, 332.64593505859375, 152.67721557617188, 254.61549377441406, 182.37371826171875, 262.41015625, 254.66915893554688, 232.82122802734375, 189.87265014648438, 191.2648162841797, 115.397216796875, 247.774169921875, 248.4126434326172, 120.35723876953125, 207.77537536621094, 37.34035110473633, 327.09368896484375, 176.76461791992188, 147.11834716796875, 102.72297668457031, 34.854373931884766, 209.44004821777344, 200.7854766845703, 216.6543731689453, 1.23737370967865, 1193.26904296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.8704268932342529, 0.3473832309246063, 0.1495087593793869, 0.23812329769134521, 0.17850947380065918, 0.254533976316452, 0.22376105189323425, 0.2285299003124237, 0.18161724507808685, 0.19555462896823883, 0.11946316063404083, 0.24775850772857666, 0.23793382942676544, 0.11462461203336716, 0.1670796126127243, 0.03747349604964256, 0.3640751838684082, 0.19524036347866058, 0.17405372858047485, 0.10189177095890045, 0.03714651241898537, 0.18566389381885529, 0.19320346415042877, 0.23296740651130676, 0.0011953806970268488, 1.046237826347351]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.862048040305387, "first_log_odds": 15.374999999999998, "later_log_odds": -0.512951959694611, "first_differing_log_odds": 15.374999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-main-bridge.npz", "array_sha256": "346c7579c0ce83ee94ceb1944e04e91483694fc88375ed21f94f6b467b8bbd2a", "seconds": 0.3155747908167541, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-baseline", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1231, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-14.500004614759163, -0.0026379302271794763], "logp": -14.502642544986342, "cache_id": 1232, "trace": []}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.375004614759163, -0.06415400106604224], "logp": -29.439158615825207, "cache_id": 1233, "trace": []}, "log_odds": -14.936516070838865, "first_log_odds": -14.875, "later_log_odds": -0.06151607083886468, "first_differing_log_odds": -14.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-baseline.npz", "array_sha256": "1f76398fc131fac44f300fd9933a0d7c89be0b2dd66e19962039ab86b5753394", "seconds": 1.192447958048433, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-noop", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.306889533996582, "match_error": -9.306889533996582, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1234, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-14.500004614759163, -0.0026379302271794763], "logp": -14.502642544986342, "cache_id": 1235, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.306889533996582, "match_error": -9.306889533996582, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.375004614759163, -0.06415400106604224], "logp": -29.439158615825207, "cache_id": 1236, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.306889533996582, "match_error": -9.306889533996582, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.936516070838865, "first_log_odds": -14.875, "later_log_odds": -0.06151607083886468, "first_differing_log_odds": -14.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-noop.npz", "array_sha256": "808caafdd7ef76412086045cb00aa1e8d0652f69ce7a34b552dbe8d1c1608dc0", "seconds": 1.196213916875422, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-j", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.306571006774902, "actual_norm": 9.306889533996582, "relative_norm": 0.0499994067073912, "target_actual_norm": 9.306889533996582, "match_error": 0.0, "scalar": 0.14269333332762718, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1237, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.625002730157114, -0.0029006812598835275], "logp": -16.627903411416998, "cache_id": 1238, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.306571006774902, "actual_norm": 9.306889533996582, "relative_norm": 0.0499994067073912, "target_actual_norm": 9.306889533996582, "match_error": 0.0, "scalar": 0.14269333332762718, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.062502730157114, -0.020799791877042503], "logp": -29.083302522034156, "cache_id": 1239, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.306571006774902, "actual_norm": 9.306889533996582, "relative_norm": 0.0499994067073912, "target_actual_norm": 9.306889533996582, "match_error": 0.0, "scalar": 0.14269333332762718, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.455399110617158, "first_log_odds": -12.4375, "later_log_odds": -0.017899110617157987, "first_differing_log_odds": -12.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-j.npz", "array_sha256": "6dbf56275295c153b2e76b5dca148f87d304f99f7c327a98e8b1c0f362a743d1", "seconds": 1.1905374168418348, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-opposite", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.30078125, "actual_norm": 9.306882858276367, "relative_norm": 0.049999370843416574, "target_actual_norm": 9.306889533996582, "match_error": -6.67572021484375e-06, "scalar": 9.30078302383481, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1240, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-13.62500578327478, -0.0022120036940520165], "logp": -13.627217786968831, "cache_id": 1241, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.30078125, "actual_norm": 9.306882858276367, "relative_norm": 0.049999370843416574, "target_actual_norm": 9.306889533996582, "match_error": -6.67572021484375e-06, "scalar": 9.30078302383481, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.31250578327478, -0.102227631715245], "logp": -29.414733414990025, "cache_id": 1242, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.30078125, "actual_norm": 9.306882858276367, "relative_norm": 0.049999370843416574, "target_actual_norm": 9.306889533996582, "match_error": -6.67572021484375e-06, "scalar": 9.30078302383481, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.787515628021193, "first_log_odds": -15.6875, "later_log_odds": -0.10001562802119324, "first_differing_log_odds": -15.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-opposite.npz", "array_sha256": "52e87501db8071e50ebce52aa1df65314b201a6b11d501872d1043a605aead95", "seconds": 1.1903720842674375, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-unrelated", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.297942161560059, "actual_norm": 9.306992530822754, "relative_norm": 0.04999996003728542, "target_actual_norm": 9.306889533996582, "match_error": 0.000102996826171875, "scalar": 9.297942786550266, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1243, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-14.25000556359693, -0.0031016726451878595], "logp": -14.25310723624212, "cache_id": 1244, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.297942161560059, "actual_norm": 9.306992530822754, "relative_norm": 0.04999996003728542, "target_actual_norm": 9.306889533996582, "match_error": 0.000102996826171875, "scalar": 9.297942786550266, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.18750556359693, -0.06327830061587918], "logp": -29.250783864212806, "cache_id": 1245, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.297942161560059, "actual_norm": 9.306992530822754, "relative_norm": 0.04999996003728542, "target_actual_norm": 9.306889533996582, "match_error": 0.000102996826171875, "scalar": 9.297942786550266, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.997676627970687, "first_log_odds": -14.937499999999998, "later_log_odds": -0.06017662797068901, "first_differing_log_odds": -14.937499999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-unrelated.npz", "array_sha256": "09604bc6907aed183bd8ec32d5db0bf2d430cc005e2f78b66dbb155caff72b11", "seconds": 1.2011218341067433, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-ordinary", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.311860084533691, "actual_norm": 9.306875228881836, "relative_norm": 0.049999329856017, "target_actual_norm": 9.306889533996582, "match_error": -1.430511474609375e-05, "scalar": 9.311859949244536, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1246, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.62500337692633, -0.002355889228174774], "logp": -15.627359266154505, "cache_id": 1247, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.311860084533691, "actual_norm": 9.306875228881836, "relative_norm": 0.049999329856017, "target_actual_norm": 9.306889533996582, "match_error": -1.430511474609375e-05, "scalar": 9.311859949244536, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.43750337692633, -0.04429143643744502], "logp": -29.481794813363773, "cache_id": 1248, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.311860084533691, "actual_norm": 9.306875228881836, "relative_norm": 0.049999329856017, "target_actual_norm": 9.306889533996582, "match_error": -1.430511474609375e-05, "scalar": 9.311859949244536, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.854435547209269, "first_log_odds": -13.8125, "later_log_odds": -0.04193554720926862, "first_differing_log_odds": -13.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-ordinary.npz", "array_sha256": "aa02e8e3b41ec18c162c128f1d48bd394c02426c45858dba1378aa505852ae21", "seconds": 1.1868069167248905, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-random0", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.31285285949707, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.312854032294126, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1249, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-14.37500528901104, -0.003203467126437304], "logp": -14.378208756137479, "cache_id": 1250, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.31285285949707, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.312854032294126, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.312505289011042, -0.04788965136046324], "logp": -29.360394940371506, "cache_id": 1251, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.31285285949707, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.312854032294126, "random_seed": 20273008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.982186184234028, "first_log_odds": -14.937500000000002, "later_log_odds": -0.044686184234025816, "first_differing_log_odds": -14.937500000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-random0.npz", "array_sha256": "64079120bc545eddd1fcd4a90b2dc12ec0c38ec52351d4b46b96a32a090f94a1", "seconds": 1.1884008329361677, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-random1", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.308894157409668, "actual_norm": 9.306904792785645, "relative_norm": 0.04999948868219034, "target_actual_norm": 9.306889533996582, "match_error": 1.52587890625e-05, "scalar": 9.308895451578792, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1252, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-16.37500208811095, -0.0023940944287320293], "logp": -16.37739618253968, "cache_id": 1253, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.308894157409668, "actual_norm": 9.306904792785645, "relative_norm": 0.04999948868219034, "target_actual_norm": 9.306889533996582, "match_error": 1.52587890625e-05, "scalar": 9.308895451578792, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.62500208811095, -0.04892697425135757], "logp": -29.67392906236231, "cache_id": 1254, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.308894157409668, "actual_norm": 9.306904792785645, "relative_norm": 0.04999948868219034, "target_actual_norm": 9.306889533996582, "match_error": 1.52587890625e-05, "scalar": 9.308895451578792, "random_seed": 20273009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.296532879822628, "first_log_odds": -13.25, "later_log_odds": -0.046532879822628104, "first_differing_log_odds": -13.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-random1.npz", "array_sha256": "107880dbd4ede7519935ef1f1a1f87de2429bbc07cef45a73650a145ba2b1891", "seconds": 1.2035728748887777, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-random2", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.292191505432129, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.292191306049062, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1255, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-15.500003352019805, -0.002783083141653611], "logp": -15.50278643516146, "cache_id": 1256, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.292191505432129, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.292191306049062, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.375003352019803, -0.052056561696978566], "logp": -29.427059913716782, "cache_id": 1257, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 186.13999938964844, "strength_one_norm": 65.22078704833984, "target_shrink_scale": 0.14269333332762718, "requested_norm": 9.292191505432129, "actual_norm": 9.306896209716797, "relative_norm": 0.04999944257136583, "target_actual_norm": 9.306889533996582, "match_error": 6.67572021484375e-06, "scalar": 9.292191306049062, "random_seed": 20273010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.924273478555323, "first_log_odds": -13.874999999999998, "later_log_odds": -0.0492734785553246, "first_differing_log_odds": -13.874999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-random2.npz", "array_sha256": "53917fa9207fd5a2db1c246df68e16b66500980e98e86b215a3d40e75c59f746", "seconds": 1.2008982910774648, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-21-paraphrase-bridge", "case_id": "eval-21", "city": "Budapest", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.<|im_end|>", "output_text": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "output_ids": [785, 11413, 1483, 304, 69595, 374, 279, 56769, 369, 396, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 39, 20538, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 868.9926147460938, 292.46826171875, 112.39495086669922, 255.5803985595703, 43.36785888671875, 265.8652038574219, 239.2639617919922, 217.76522827148438, 190.707763671875, 190.46897888183594, 139.59658813476562, 250.3045196533203, 207.31539916992188, 232.6864471435547, 154.07464599609375, 42.10622787475586, 101.03919982910156, 2.2205111980438232, 208.91348266601562, 209.66122436523438, 274.2513427734375, 1.3002431392669678, 800.5125122070312]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 869.0440063476562, 292.4791259765625, 112.37303924560547, 255.56370544433594, 43.42518615722656, 265.8781433105469, 239.22396850585938, 217.7394561767578, 190.76480102539062, 190.4570770263672, 139.5504150390625, 250.28997802734375, 207.28973388671875, 232.67648315429688, 154.0687255859375, 42.19462203979492, 100.98188781738281, 2.275496006011963, 208.88833618164062, 209.65533447265625, 274.27545166015625, 1.1949890851974487, 800.435791015625]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.8282789587974548, 0.2828277051448822, 0.1021961197257042, 0.2416771799325943, 0.03954090550541878, 0.25685030221939087, 0.20524361729621887, 0.20804962515830994, 0.17982472479343414, 0.19248805940151215, 0.143428772687912, 0.24483099579811096, 0.19632376730442047, 0.23966526985168457, 0.17334508895874023, 0.048317454755306244, 0.09863375127315521, 0.002419885713607073, 0.18265677988529205, 0.1986924707889557, 0.29004132747650146, 0.001153364428319037, 0.8206718564033508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1258, "cached": true, "scoring": {"normalized": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **HUF**.", "bare_format": false, "malformed": true, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Budapest is the Hungarian forint, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Hungarian", "start": 37, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-29.250002482536036, -0.0011826218480581769], "logp": -29.251185104384096, "cache_id": 1259, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 868.9926147460938, 292.46826171875, 112.39495086669922, 255.5803985595703, 43.36785888671875, 265.8652038574219, 239.2639617919922, 217.76522827148438, 190.707763671875, 190.46897888183594, 139.59658813476562, 250.3045196533203, 207.31539916992188, 232.6864471435547, 154.07464599609375, 42.10622787475586, 101.03919982910156, 2.2205111980438232, 208.91348266601562, 209.66122436523438, 274.2513427734375, 1.3002431392669678, 800.5125122070312]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 869.0440063476562, 292.4791259765625, 112.37303924560547, 255.56370544433594, 43.42518615722656, 265.8781433105469, 239.22396850585938, 217.7394561767578, 190.76480102539062, 190.4570770263672, 139.5504150390625, 250.28997802734375, 207.28973388671875, 232.67648315429688, 154.0687255859375, 42.19462203979492, 100.98188781738281, 2.275496006011963, 208.88833618164062, 209.65533447265625, 274.27545166015625, 1.1949890851974487, 800.435791015625]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.8282789587974548, 0.2828277051448822, 0.1021961197257042, 0.2416771799325943, 0.03954090550541878, 0.25685030221939087, 0.20524361729621887, 0.20804962515830994, 0.17982472479343414, 0.19248805940151215, 0.143428772687912, 0.24483099579811096, 0.19632376730442047, 0.23966526985168457, 0.17334508895874023, 0.048317454755306244, 0.09863375127315521, 0.002419885713607073, 0.18265677988529205, 0.1986924707889557, 0.29004132747650146, 0.001153364428319037, 0.8206718564033508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-18.875002482536036, -0.11754847071051251], "logp": -18.99255095324655, "cache_id": 1260, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 868.9926147460938, 292.46826171875, 112.39495086669922, 255.5803985595703, 43.36785888671875, 265.8652038574219, 239.2639617919922, 217.76522827148438, 190.707763671875, 190.46897888183594, 139.59658813476562, 250.3045196533203, 207.31539916992188, 232.6864471435547, 154.07464599609375, 42.10622787475586, 101.03919982910156, 2.2205111980438232, 208.91348266601562, 209.66122436523438, 274.2513427734375, 1.3002431392669678, 800.5125122070312]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 869.0440063476562, 292.4791259765625, 112.37303924560547, 255.56370544433594, 43.42518615722656, 265.8781433105469, 239.22396850585938, 217.7394561767578, 190.76480102539062, 190.4570770263672, 139.5504150390625, 250.28997802734375, 207.28973388671875, 232.67648315429688, 154.0687255859375, 42.19462203979492, 100.98188781738281, 2.275496006011963, 208.88833618164062, 209.65533447265625, 274.27545166015625, 1.1949890851974487, 800.435791015625]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.8282789587974548, 0.2828277051448822, 0.1021961197257042, 0.2416771799325943, 0.03954090550541878, 0.25685030221939087, 0.20524361729621887, 0.20804962515830994, 0.17982472479343414, 0.19248805940151215, 0.143428772687912, 0.24483099579811096, 0.19632376730442047, 0.23966526985168457, 0.17334508895874023, 0.048317454755306244, 0.09863375127315521, 0.002419885713607073, 0.18265677988529205, 0.1986924707889557, 0.29004132747650146, 0.001153364428319037, 0.8206718564033508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 10.258634151137546, "first_log_odds": 10.375, "later_log_odds": -0.11636584886245416, "first_differing_log_odds": 10.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-21-paraphrase-bridge.npz", "array_sha256": "c53e80f881de6525867c7f9574cac3e288874ad2ec8c5c2fd39c8bdc29ec695c", "seconds": 1.185341542121023, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-baseline", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1261, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.011119552738110192, -0.012097067109567608], "logp": -0.0232166198476778, "cache_id": 1262, "trace": []}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.13611955273811, -1.7286563597775837], "logp": -28.864775912515693, "cache_id": 1263, "trace": []}, "log_odds": -28.841559292668016, "first_log_odds": -27.125, "later_log_odds": -1.7165592926680162, "first_differing_log_odds": -27.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-baseline.npz", "array_sha256": "17c4543edf7e9d57456a625beb0e5ee8ca3cad13573534f7edeac8cf9c5cad11", "seconds": 0.3808342907577753, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-noop", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.316707611083984, "match_error": -9.316707611083984, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1264, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.011119552738110192, -0.012097067109567608], "logp": -0.0232166198476778, "cache_id": 1265, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.316707611083984, "match_error": -9.316707611083984, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.13611955273811, -1.7286563597775837], "logp": -28.864775912515693, "cache_id": 1266, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.316707611083984, "match_error": -9.316707611083984, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.841559292668016, "first_log_odds": -27.125, "later_log_odds": -1.7165592926680162, "first_differing_log_odds": -27.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-noop.npz", "array_sha256": "edae42d1d5224b013e16d9aa41afbb4f703235ce2ca0da9bfad981fd37e12c8a", "seconds": 0.3385632080025971, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-j", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.30916976928711, "actual_norm": 9.316707611083984, "relative_norm": 0.04999988944957517, "target_actual_norm": 9.316707611083984, "match_error": 0.0, "scalar": 0.22611358013317523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1267, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.018333254697451774, -0.014825062678219546], "logp": -0.03315831737567132, "cache_id": 1268, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.30916976928711, "actual_norm": 9.316707611083984, "relative_norm": 0.04999988944957517, "target_actual_norm": 9.316707611083984, "match_error": 0.0, "scalar": 0.22611358013317523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-24.330833254697453, -1.2840989313453317], "logp": -25.614932186042786, "cache_id": 1269, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.30916976928711, "actual_norm": 9.316707611083984, "relative_norm": 0.04999988944957517, "target_actual_norm": 9.316707611083984, "match_error": 0.0, "scalar": 0.22611358013317523, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -25.581773868667113, "first_log_odds": -24.3125, "later_log_odds": -1.2692738686671134, "first_differing_log_odds": -24.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-j.npz", "array_sha256": "c4120a525da8fab1426a939071b1cd3ee9ede2656f29626d5cf81106b344c3f6", "seconds": 0.3419391671195626, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-opposite", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.310876846313477, "actual_norm": 9.316374778747559, "relative_norm": 0.049998103241322076, "target_actual_norm": 9.316707611083984, "match_error": -0.00033283233642578125, "scalar": 9.310878982360009, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1270, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.005275812526010062, -0.008369965434068982], "logp": -0.013645777960079045, "cache_id": 1271, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.310876846313477, "actual_norm": 9.316374778747559, "relative_norm": 0.049998103241322076, "target_actual_norm": 9.316707611083984, "match_error": -0.00033283233642578125, "scalar": 9.310878982360009, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-29.31777581252601, -2.019183346028628], "logp": -31.336959158554635, "cache_id": 1272, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.310876846313477, "actual_norm": 9.316374778747559, "relative_norm": 0.049998103241322076, "target_actual_norm": 9.316707611083984, "match_error": -0.00033283233642578125, "scalar": 9.310878982360009, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -31.323313380594556, "first_log_odds": -29.3125, "later_log_odds": -2.0108133805945556, "first_differing_log_odds": -29.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-opposite.npz", "array_sha256": "cda22b88f8242df46f0e41088c393a6469be84dd2c1447b2e392e979ddef8450", "seconds": 0.34063816675916314, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-unrelated", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.306754112243652, "actual_norm": 9.316712379455566, "relative_norm": 0.049999915039951284, "target_actual_norm": 9.316707611083984, "match_error": 4.76837158203125e-06, "scalar": 9.306756293750368, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1273, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.008683706461422249, -0.011453364833869486], "logp": -0.020137071295291735, "cache_id": 1274, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.306754112243652, "actual_norm": 9.316712379455566, "relative_norm": 0.049999915039951284, "target_actual_norm": 9.316707611083984, "match_error": 4.76837158203125e-06, "scalar": 9.306756293750368, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.696183706461422, -1.449322765199586], "logp": -29.145506471661008, "cache_id": 1275, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.306754112243652, "actual_norm": 9.316712379455566, "relative_norm": 0.049999915039951284, "target_actual_norm": 9.316707611083984, "match_error": 4.76837158203125e-06, "scalar": 9.306756293750368, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.125369400365717, "first_log_odds": -27.6875, "later_log_odds": -1.4378694003657166, "first_differing_log_odds": -27.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-unrelated.npz", "array_sha256": "1df30c30c7671e1be647c81613edbcc78804dc58932f2520e52b8b37ceec9966", "seconds": 0.3444067919626832, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-ordinary", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.31692123413086, "actual_norm": 9.315945625305176, "relative_norm": 0.04999580010747139, "target_actual_norm": 9.316707611083984, "match_error": -0.0007619857788085938, "scalar": 9.316920853598276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1276, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.008678685405892493, -0.012071110913456883], "logp": -0.020749796319349374, "cache_id": 1277, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.31692123413086, "actual_norm": 9.315945625305176, "relative_norm": 0.04999580010747139, "target_actual_norm": 9.316707611083984, "match_error": -0.0007619857788085938, "scalar": 9.316920853598276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.071178685405894, -1.625927515580467], "logp": -28.69710620098636, "cache_id": 1278, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.31692123413086, "actual_norm": 9.315945625305176, "relative_norm": 0.04999580010747139, "target_actual_norm": 9.316707611083984, "match_error": -0.0007619857788085938, "scalar": 9.316920853598276, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.67635640466701, "first_log_odds": -27.0625, "later_log_odds": -1.6138564046670112, "first_differing_log_odds": -27.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-ordinary.npz", "array_sha256": "fda17a5efd423879b4588a4e9271d49f03a2a9f7edd108b1ff16992c48019c0c", "seconds": 0.33501208294183016, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-random0", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.318554878234863, "actual_norm": 9.316211700439453, "relative_norm": 0.049997228050458815, "target_actual_norm": 9.316707611083984, "match_error": -0.00049591064453125, "scalar": 9.318555712874513, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1279, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.006770587558160084, -0.010374989995537803], "logp": -0.01714557755369789, "cache_id": 1280, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.318554878234863, "actual_norm": 9.316211700439453, "relative_norm": 0.049997228050458815, "target_actual_norm": 9.316707611083984, "match_error": -0.00049591064453125, "scalar": 9.318555712874513, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.69427058755816, -1.6513050499485642], "logp": -29.345575637506727, "cache_id": 1281, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.318554878234863, "actual_norm": 9.316211700439453, "relative_norm": 0.049997228050458815, "target_actual_norm": 9.316707611083984, "match_error": -0.00049591064453125, "scalar": 9.318555712874513, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.32843005995303, "first_log_odds": -27.6875, "later_log_odds": -1.6409300599530283, "first_differing_log_odds": -27.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-random0.npz", "array_sha256": "d937a5581deb67d243f07b36fda6860f65d3708b6a4fb54f6d867020579c9be0", "seconds": 0.34184479201212525, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-random1", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.303780555725098, "actual_norm": 9.316689491271973, "relative_norm": 0.049999792206145914, "target_actual_norm": 9.316707611083984, "match_error": -1.811981201171875e-05, "scalar": 9.303779783655045, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1282, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.01114720303380579, -0.01253795542139412], "logp": -0.02368515845519991, "cache_id": 1283, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.303780555725098, "actual_norm": 9.316689491271973, "relative_norm": 0.049999792206145914, "target_actual_norm": 9.316707611083984, "match_error": -1.811981201171875e-05, "scalar": 9.303779783655045, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.761147203033804, -1.661422660007477], "logp": -28.422569863041282, "cache_id": 1284, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.303780555725098, "actual_norm": 9.316689491271973, "relative_norm": 0.049999792206145914, "target_actual_norm": 9.316707611083984, "match_error": -1.811981201171875e-05, "scalar": 9.303779783655045, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.39888470458608, "first_log_odds": -26.75, "later_log_odds": -1.6488847045860808, "first_differing_log_odds": -26.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-random1.npz", "array_sha256": "6302aec9e1701be3f813c4026d17e27505e2226aa8493db951cd50bd4764db8f", "seconds": 0.3391730831936002, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-random2", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "HUF<|im_end|>", "output_text": "HUF", "output_ids": [39, 20538, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.313864707946777, "actual_norm": 9.316672325134277, "relative_norm": 0.04999970008079189, "target_actual_norm": 9.316707611083984, "match_error": -3.528594970703125e-05, "scalar": 9.313864377560094, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1285, "cached": true, "scoring": {"normalized": "HUF", "bare_format": true, "malformed": false, "first_code": "HUF", "all_codes": ["HUF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-0.008685886157983025, -0.012942889321246927], "logp": -0.02162877547922995, "cache_id": 1286, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.313864707946777, "actual_norm": 9.316672325134277, "relative_norm": 0.04999970008079189, "target_actual_norm": 9.316707611083984, "match_error": -3.528594970703125e-05, "scalar": 9.313864377560094, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.196185886157984, -1.653077762029685], "logp": -28.84926364818767, "cache_id": 1287, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 186.33456420898438, "strength_one_norm": 41.17033004760742, "target_shrink_scale": 0.22611358013317523, "requested_norm": 9.313864707946777, "actual_norm": 9.316672325134277, "relative_norm": 0.04999970008079189, "target_actual_norm": 9.316707611083984, "match_error": -3.528594970703125e-05, "scalar": 9.313864377560094, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.82763487270844, "first_log_odds": -27.1875, "later_log_odds": -1.6401348727084404, "first_differing_log_odds": -27.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-random2.npz", "array_sha256": "9cf020123319900324a43b9bf7f0c1ea1e8d351132f31812e7214d3f0b5e2894", "seconds": 0.3378822081722319, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-main-bridge", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 328.9560546875, 856.3941650390625, 313.5442810058594, 145.23341369628906, 233.56588745117188, 186.6128387451172, 259.04034423828125, 249.94139099121094, 242.34339904785156, 176.78736877441406, 182.47108459472656, 114.34420013427734, 230.49046325683594, 227.17135620117188, 126.5381088256836, 208.23306274414062, 40.336631774902344, 301.5462646484375, 162.9069366455078, 116.08495330810547, 98.84117889404297, 35.51190948486328, 196.76148986816406, 185.59039306640625, 211.5358123779297, 3.021839141845703, 1053.28173828125]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 328.89849853515625, 856.4501953125, 313.52435302734375, 145.2208709716797, 233.54507446289062, 186.63990783691406, 259.0456237792969, 249.88037109375, 242.3294219970703, 176.76605224609375, 182.5093994140625, 114.36962890625, 230.5296173095703, 227.2155303955078, 126.52469635009766, 208.23716735839844, 40.35858154296875, 301.5702819824219, 162.94020080566406, 116.10001373291016, 98.82185363769531, 35.54222869873047, 196.81985473632812, 185.60397338867188, 211.57923889160156, 3.093644857406616, 1053.3514404296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.2885664105415344, 0.8208276629447937, 0.32816892862319946, 0.1428833305835724, 0.22161677479743958, 0.18095198273658752, 0.2518314719200134, 0.22018469870090485, 0.23981072008609772, 0.16887226700782776, 0.18512886762619019, 0.11954405158758163, 0.23400092124938965, 0.22027264535427094, 0.11984269320964813, 0.16874463856220245, 0.040713321417570114, 0.33676406741142273, 0.18023042380809784, 0.1393691748380661, 0.09901324659585953, 0.03812229260802269, 0.17381678521633148, 0.18070325255393982, 0.22958694398403168, 0.002987110987305641, 0.9302873015403748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}], "cache_id": 1288, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.036285182096783, -0.006827138817034992], "logp": -19.043112320913817, "cache_id": 1289, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 328.9560546875, 856.3941650390625, 313.5442810058594, 145.23341369628906, 233.56588745117188, 186.6128387451172, 259.04034423828125, 249.94139099121094, 242.34339904785156, 176.78736877441406, 182.47108459472656, 114.34420013427734, 230.49046325683594, 227.17135620117188, 126.5381088256836, 208.23306274414062, 40.336631774902344, 301.5462646484375, 162.9069366455078, 116.08495330810547, 98.84117889404297, 35.51190948486328, 196.76148986816406, 185.59039306640625, 211.5358123779297, 3.021839141845703, 1053.28173828125]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 328.89849853515625, 856.4501953125, 313.52435302734375, 145.2208709716797, 233.54507446289062, 186.63990783691406, 259.0456237792969, 249.88037109375, 242.3294219970703, 176.76605224609375, 182.5093994140625, 114.36962890625, 230.5296173095703, 227.2155303955078, 126.52469635009766, 208.23716735839844, 40.35858154296875, 301.5702819824219, 162.94020080566406, 116.10001373291016, 98.82185363769531, 35.54222869873047, 196.81985473632812, 185.60397338867188, 211.57923889160156, 3.093644857406616, 1053.3514404296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.2885664105415344, 0.8208276629447937, 0.32816892862319946, 0.1428833305835724, 0.22161677479743958, 0.18095198273658752, 0.2518314719200134, 0.22018469870090485, 0.23981072008609772, 0.16887226700782776, 0.18512886762619019, 0.11954405158758163, 0.23400092124938965, 0.22027264535427094, 0.11984269320964813, 0.16874463856220245, 0.040713321417570114, 0.33676406741142273, 0.18023042380809784, 0.1393691748380661, 0.09901324659585953, 0.03812229260802269, 0.17381678521633148, 0.18070325255393982, 0.22958694398403168, 0.002987110987305641, 0.9302873015403748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-2.848785182096784, -2.0537178801707427], "logp": -4.902503062267527, "cache_id": 1290, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 25.11289405822754, 179.49205017089844, 90.64324951171875, 237.5504608154297, 295.43280029296875, 328.9560546875, 856.3941650390625, 313.5442810058594, 145.23341369628906, 233.56588745117188, 186.6128387451172, 259.04034423828125, 249.94139099121094, 242.34339904785156, 176.78736877441406, 182.47108459472656, 114.34420013427734, 230.49046325683594, 227.17135620117188, 126.5381088256836, 208.23306274414062, 40.336631774902344, 301.5462646484375, 162.9069366455078, 116.08495330810547, 98.84117889404297, 35.51190948486328, 196.76148986816406, 185.59039306640625, 211.5358123779297, 3.021839141845703, 1053.28173828125]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 25.110986709594727, 179.48890686035156, 90.70387268066406, 237.57574462890625, 295.42974853515625, 328.89849853515625, 856.4501953125, 313.52435302734375, 145.2208709716797, 233.54507446289062, 186.63990783691406, 259.0456237792969, 249.88037109375, 242.3294219970703, 176.76605224609375, 182.5093994140625, 114.36962890625, 230.5296173095703, 227.2155303955078, 126.52469635009766, 208.23716735839844, 40.35858154296875, 301.5702819824219, 162.94020080566406, 116.10001373291016, 98.82185363769531, 35.54222869873047, 196.81985473632812, 185.60397338867188, 211.57923889160156, 3.093644857406616, 1053.3514404296875]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.023051002994179726, 0.15748022496700287, 0.07379241287708282, 0.2033458948135376, 0.2603329122066498, 0.2885664105415344, 0.8208276629447937, 0.32816892862319946, 0.1428833305835724, 0.22161677479743958, 0.18095198273658752, 0.2518314719200134, 0.22018469870090485, 0.23981072008609772, 0.16887226700782776, 0.18512886762619019, 0.11954405158758163, 0.23400092124938965, 0.22027264535427094, 0.11984269320964813, 0.16874463856220245, 0.040713321417570114, 0.33676406741142273, 0.18023042380809784, 0.1393691748380661, 0.09901324659585953, 0.03812229260802269, 0.17381678521633148, 0.18070325255393982, 0.22958694398403168, 0.002987110987305641, 0.9302873015403748]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.14060925864629, "first_log_odds": 16.1875, "later_log_odds": -2.0468907413537103, "first_differing_log_odds": 16.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-main-bridge.npz", "array_sha256": "7d70d1c10fae85a61c7ef35bfe789071d185e8e7e3337ffaf73e4e694d4f6827", "seconds": 0.3179693752899766, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-baseline", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1291, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-18.87500273018971, -0.049970593471145804], "logp": -18.924973323660854, "cache_id": 1292, "trace": []}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.43750273018971, -1.2363159559806598], "logp": -27.67381868617037, "cache_id": 1293, "trace": []}, "log_odds": -8.748845362509517, "first_log_odds": -7.5625, "later_log_odds": -1.1863453625095168, "first_differing_log_odds": -7.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-baseline.npz", "array_sha256": "1588a075b801146d3594ac44852aacea8cf76a7e39a28436c0542cf22785435a", "seconds": 1.2950478750281036, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-noop", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.177395820617676, "match_error": -9.177395820617676, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1294, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-18.87500273018971, -0.049970593471145804], "logp": -18.924973323660854, "cache_id": 1295, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.177395820617676, "match_error": -9.177395820617676, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.43750273018971, -1.2363159559806598], "logp": -27.67381868617037, "cache_id": 1296, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.177395820617676, "match_error": -9.177395820617676, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.748845362509517, "first_log_odds": -7.5625, "later_log_odds": -1.1863453625095168, "first_differing_log_odds": -7.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-noop.npz", "array_sha256": "073defc34469caaf86c25a920075725f57eac26f76a588b35ae1bc54dcf531b5", "seconds": 1.2903241249732673, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-j", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.165788650512695, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 0.2721259444588646, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1297, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.750003674901325, -0.1301634393106283], "logp": -19.880167114211954, "cache_id": 1298, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.165788650512695, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 0.2721259444588646, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.937503674901325, -1.05292116355273], "logp": -26.990424838454054, "cache_id": 1299, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.165788650512695, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 0.2721259444588646, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.1102577242421, "first_log_odds": -6.1875, "later_log_odds": -0.9227577242420999, "first_differing_log_odds": -6.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-j.npz", "array_sha256": "8628c117a42f5f7bc6f83f0ad26402a2a1e285513fdd68386c336ab9d58dfcfb", "seconds": 1.3024026658385992, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-opposite", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.168923377990723, "actual_norm": 9.17618179321289, "relative_norm": 0.04999336187657131, "target_actual_norm": 9.177395820617676, "match_error": -0.0012140274047851562, "scalar": 9.168923645825998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1300, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-17.62500263342141, -0.01936976991322944], "logp": -17.64437240333464, "cache_id": 1301, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.168923377990723, "actual_norm": 9.17618179321289, "relative_norm": 0.04999336187657131, "target_actual_norm": 9.177395820617676, "match_error": -0.0012140274047851562, "scalar": 9.168923645825998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.81250263342141, -1.3450779313874723], "logp": -28.15758056480888, "cache_id": 1302, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.168923377990723, "actual_norm": 9.17618179321289, "relative_norm": 0.04999336187657131, "target_actual_norm": 9.177395820617676, "match_error": -0.0012140274047851562, "scalar": 9.168923645825998, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -10.513208161474243, "first_log_odds": -9.1875, "later_log_odds": -1.3257081614742425, "first_differing_log_odds": -9.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-opposite.npz", "array_sha256": "9bdc04df3ace8ec3781d569c086fd84a49283dd20013a804866f6c7d5a2218e7", "seconds": 1.2842287919484079, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-unrelated", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.169798851013184, "actual_norm": 9.177298545837402, "relative_norm": 0.04999944613027747, "target_actual_norm": 9.177395820617676, "match_error": -9.72747802734375e-05, "scalar": 9.16979887049456, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1303, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-18.875002157126215, -0.03268175007494492], "logp": -18.90768390720116, "cache_id": 1304, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.169798851013184, "actual_norm": 9.177298545837402, "relative_norm": 0.04999944613027747, "target_actual_norm": 9.177395820617676, "match_error": -9.72747802734375e-05, "scalar": 9.16979887049456, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.687502157126215, -1.0828526732574177], "logp": -27.77035483038363, "cache_id": 1305, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.169798851013184, "actual_norm": 9.177298545837402, "relative_norm": 0.04999944613027747, "target_actual_norm": 9.177395820617676, "match_error": -9.72747802734375e-05, "scalar": 9.16979887049456, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.86267092318247, "first_log_odds": -7.8125, "later_log_odds": -1.0501709231824705, "first_differing_log_odds": -7.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-unrelated.npz", "array_sha256": "1481b30fc9297a7ac353b7356a7fdfbba66c1b1715646927ca565a1ee95a4c9b", "seconds": 1.2947270423173904, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-ordinary", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.175138473510742, "actual_norm": 9.17733097076416, "relative_norm": 0.04999962278666177, "target_actual_norm": 9.177395820617676, "match_error": -6.4849853515625e-05, "scalar": 9.175137740972787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1306, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.125002405252104, -0.05916756552317051], "logp": -19.184169970775276, "cache_id": 1307, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.175138473510742, "actual_norm": 9.17733097076416, "relative_norm": 0.04999962278666177, "target_actual_norm": 9.177395820617676, "match_error": -6.4849853515625e-05, "scalar": 9.175137740972787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.750002405252104, -1.2364941422109883], "logp": -27.98649654746309, "cache_id": 1308, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.175138473510742, "actual_norm": 9.17733097076416, "relative_norm": 0.04999962278666177, "target_actual_norm": 9.177395820617676, "match_error": -6.4849853515625e-05, "scalar": 9.175137740972787, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.802326576687815, "first_log_odds": -7.625, "later_log_odds": -1.1773265766878147, "first_differing_log_odds": -7.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-ordinary.npz", "array_sha256": "bda818d28471aed2de51a0832a776274a758bb5f84126ed7f5443149576431aa", "seconds": 1.3012375831604004, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-random0", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.179354667663574, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 9.179356323875254, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1309, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.250001832764774, -0.055839020124627096], "logp": -19.3058408528894, "cache_id": 1310, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.179354667663574, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 9.179356323875254, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.812501832764774, -1.2596573101852648], "logp": -28.07215914295004, "cache_id": 1311, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.179354667663574, "actual_norm": 9.177395820617676, "relative_norm": 0.04999997609943036, "target_actual_norm": 9.177395820617676, "match_error": 0.0, "scalar": 9.179356323875254, "random_seed": 20273108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.766318290060639, "first_log_odds": -7.5625, "later_log_odds": -1.2038182900606387, "first_differing_log_odds": -7.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-random0.npz", "array_sha256": "09431474604d993c4aaba59f1cfca3d21aa958fa14d74356f2601355076b92da", "seconds": 1.271753333043307, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-random1", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.171725273132324, "actual_norm": 9.17735481262207, "relative_norm": 0.049999752681061986, "target_actual_norm": 9.177395820617676, "match_error": -4.100799560546875e-05, "scalar": 9.171724364765396, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1312, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.625001683782127, -0.04728521885910687], "logp": -19.672286902641233, "cache_id": 1313, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.171725273132324, "actual_norm": 9.17735481262207, "relative_norm": 0.049999752681061986, "target_actual_norm": 9.177395820617676, "match_error": -4.100799560546875e-05, "scalar": 9.171724364765396, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.125001683782127, -1.4839916882699988], "logp": -28.608993372052126, "cache_id": 1314, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.171725273132324, "actual_norm": 9.17735481262207, "relative_norm": 0.049999752681061986, "target_actual_norm": 9.177395820617676, "match_error": -4.100799560546875e-05, "scalar": 9.171724364765396, "random_seed": 20273109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.936706469410893, "first_log_odds": -7.5, "later_log_odds": -1.4367064694108933, "first_differing_log_odds": -7.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-random1.npz", "array_sha256": "e2ef7413dd89aedd3e2c32a7fd9fd534f2dfdd9b1e6cb371bb368373d87b880e", "seconds": 1.2856029169633985, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-random2", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.184956550598145, "actual_norm": 9.177323341369629, "relative_norm": 0.0499995812204537, "target_actual_norm": 9.177395820617676, "match_error": -7.2479248046875e-05, "scalar": 9.184957761754049, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1315, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-19.125002702531237, -0.042611757104954774], "logp": -19.16761445963619, "cache_id": 1316, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.184956550598145, "actual_norm": 9.177323341369629, "relative_norm": 0.0499995812204537, "target_actual_norm": 9.177395820617676, "match_error": -7.2479248046875e-05, "scalar": 9.184957761754049, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.687502702531237, -1.1620073071990646], "logp": -27.849510009730302, "cache_id": 1317, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 183.54800415039062, "strength_one_norm": 33.68215560913086, "target_shrink_scale": 0.2721259444588646, "requested_norm": 9.184956550598145, "actual_norm": 9.177323341369629, "relative_norm": 0.0499995812204537, "target_actual_norm": 9.177395820617676, "match_error": -7.2479248046875e-05, "scalar": 9.184957761754049, "random_seed": 20273110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.681895550094112, "first_log_odds": -7.5625, "later_log_odds": -1.1193955500941115, "first_differing_log_odds": -7.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-random2.npz", "array_sha256": "2232f965aa8ccec8e6315d1a812253b29d2b49316d7b7a89ae1933b7f870b5e8", "seconds": 1.2857877919450402, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-22-paraphrase-bridge", "case_id": "eval-22", "city": "Pecs", "country": "Hungary", "code": "HUF", "target_country": "Israel", "target_code": "ILS", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.<|im_end|>", "output_text": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "output_ids": [785, 11413, 1483, 304, 393, 53717, 11, 58719, 11, 374, 279, 88624, 27022, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 33, 3825, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 310.3937683105469, 682.8738403320312, 318.8656005859375, 100.15521240234375, 249.72506713867188, 61.040794372558594, 242.39186096191406, 238.95172119140625, 235.8621368408203, 169.3126220703125, 176.4915008544922, 130.66017150878906, 223.2441864013672, 201.8047332763672, 231.55055236816406, 144.15310668945312, 22.392675399780273, 109.0531005859375, 2.0166015625, 210.8234100341797, 200.61367797851562, 295.9293212890625, 4.236355304718018, 609.48193359375]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 310.4244689941406, 682.73681640625, 318.8852233886719, 100.22270965576172, 249.6954345703125, 61.05158615112305, 242.39804077148438, 238.916748046875, 235.88751220703125, 169.27207946777344, 176.45994567871094, 130.64828491210938, 223.29776000976562, 201.79083251953125, 231.579833984375, 144.08863830566406, 22.432506561279297, 109.0844497680664, 2.060349702835083, 210.83636474609375, 200.6468048095703, 295.94488525390625, 4.369100093841553, 609.5209350585938]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.2775454819202423, 0.6515674591064453, 0.32692238688468933, 0.09351275861263275, 0.23301754891872406, 0.056410957127809525, 0.2402695119380951, 0.20663249492645264, 0.22442756593227386, 0.1605367660522461, 0.1801670342683792, 0.1359451562166214, 0.22221454977989197, 0.19304300844669342, 0.24865466356277466, 0.1633373349905014, 0.025868456810712814, 0.10706613957881927, 0.002197457477450371, 0.18465690314769745, 0.19156867265701294, 0.31096500158309937, 0.004222321789711714, 0.6067671775817871]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1318, "cached": true, "scoring": {"normalized": "The currency used in Pecs, Bulgaria, is the Bulgarian Lev, and its three-letter ISO 4217 code is **BGL**.", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [39, 20538], "tokens": ["H", "UF"], "log_probs": [-27.062522927604352, -0.033477460486017394], "logp": -27.09600038809037, "cache_id": 1319, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 310.3937683105469, 682.8738403320312, 318.8656005859375, 100.15521240234375, 249.72506713867188, 61.040794372558594, 242.39186096191406, 238.95172119140625, 235.8621368408203, 169.3126220703125, 176.4915008544922, 130.66017150878906, 223.2441864013672, 201.8047332763672, 231.55055236816406, 144.15310668945312, 22.392675399780273, 109.0531005859375, 2.0166015625, 210.8234100341797, 200.61367797851562, 295.9293212890625, 4.236355304718018, 609.48193359375]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 310.4244689941406, 682.73681640625, 318.8852233886719, 100.22270965576172, 249.6954345703125, 61.05158615112305, 242.39804077148438, 238.916748046875, 235.88751220703125, 169.27207946777344, 176.45994567871094, 130.64828491210938, 223.29776000976562, 201.79083251953125, 231.579833984375, 144.08863830566406, 22.432506561279297, 109.0844497680664, 2.060349702835083, 210.83636474609375, 200.6468048095703, 295.94488525390625, 4.369100093841553, 609.5209350585938]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.2775454819202423, 0.6515674591064453, 0.32692238688468933, 0.09351275861263275, 0.23301754891872406, 0.056410957127809525, 0.2402695119380951, 0.20663249492645264, 0.22442756593227386, 0.1605367660522461, 0.1801670342683792, 0.1359451562166214, 0.22221454977989197, 0.19304300844669342, 0.24865466356277466, 0.1633373349905014, 0.025868456810712814, 0.10706613957881927, 0.002197457477450371, 0.18465690314769745, 0.19156867265701294, 0.31096500158309937, 0.004222321789711714, 0.6067671775817871]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-11.125022927604354, -1.4238376829303352], "logp": -12.548860610534689, "cache_id": 1320, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[235.37191772460938, 27.425519943237305, 11.181127548217773, 124.10945892333984, 63.805580139160156, 214.60902404785156, 227.5631561279297, 271.38531494140625, 310.3937683105469, 682.8738403320312, 318.8656005859375, 100.15521240234375, 249.72506713867188, 61.040794372558594, 242.39186096191406, 238.95172119140625, 235.8621368408203, 169.3126220703125, 176.4915008544922, 130.66017150878906, 223.2441864013672, 201.8047332763672, 231.55055236816406, 144.15310668945312, 22.392675399780273, 109.0531005859375, 2.0166015625, 210.8234100341797, 200.61367797851562, 295.9293212890625, 4.236355304718018, 609.48193359375]], "actual_norm": [[235.36102294921875, 27.44576072692871, 11.182682991027832, 124.07695770263672, 63.85739517211914, 214.610107421875, 227.59376525878906, 271.36151123046875, 310.4244689941406, 682.73681640625, 318.8852233886719, 100.22270965576172, 249.6954345703125, 61.05158615112305, 242.39804077148438, 238.916748046875, 235.88751220703125, 169.27207946777344, 176.45994567871094, 130.64828491210938, 223.29776000976562, 201.79083251953125, 231.579833984375, 144.08863830566406, 22.432506561279297, 109.0844497680664, 2.060349702835083, 210.83636474609375, 200.6468048095703, 295.94488525390625, 4.369100093841553, 609.5209350585938]], "relative_norm": [[0.013736816123127937, 0.029113881289958954, 0.012371533550322056, 0.10818213224411011, 0.05812780559062958, 0.18643951416015625, 0.20241932570934296, 0.24105212092399597, 0.2775454819202423, 0.6515674591064453, 0.32692238688468933, 0.09351275861263275, 0.23301754891872406, 0.056410957127809525, 0.2402695119380951, 0.20663249492645264, 0.22442756593227386, 0.1605367660522461, 0.1801670342683792, 0.1359451562166214, 0.22221454977989197, 0.19304300844669342, 0.24865466356277466, 0.1633373349905014, 0.025868456810712814, 0.10706613957881927, 0.002197457477450371, 0.18465690314769745, 0.19156867265701294, 0.31096500158309937, 0.004222321789711714, 0.6067671775817871]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.547139777555682, "first_log_odds": 15.937499999999998, "later_log_odds": -1.3903602224443166, "first_differing_log_odds": 15.937499999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-22-paraphrase-bridge.npz", "array_sha256": "2e73df4e8adeba46a4fb16977f7113cf567cf205e8d46765ba65236ddceeb7e2", "seconds": 1.2855896251276135, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-baseline", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1321, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.04292517067716402, -0.005150793758624089], "logp": -0.048075964435788106, "cache_id": 1322, "trace": []}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-13.167925170677163, -2.1082645623417926e-05], "logp": -13.167946253322787, "cache_id": 1323, "trace": []}, "log_odds": -13.119870288886998, "first_log_odds": -13.125, "later_log_odds": 0.005129711113001534, "first_differing_log_odds": -13.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-baseline.npz", "array_sha256": "e07ff962e7b61237bb5c64234ef4af9a664fc28a47917ac50fc11c08fa75a1c1", "seconds": 0.3888101247139275, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457468928, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-noop", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437359809875488, "match_error": -9.437359809875488, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1324, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.04292517067716402, -0.005150793758624089], "logp": -0.048075964435788106, "cache_id": 1325, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437359809875488, "match_error": -9.437359809875488, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-13.167925170677163, -2.1082645623417926e-05], "logp": -13.167946253322787, "cache_id": 1326, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.437359809875488, "match_error": -9.437359809875488, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.119870288886998, "first_log_odds": -13.125, "later_log_odds": 0.005129711113001534, "first_differing_log_odds": -13.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-noop.npz", "array_sha256": "143c0675010dca293d264aee35fe72bc9ad06330f9c09ca748ccb123ef250ed6", "seconds": 0.34903029073029757, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457501696, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-j", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43132209777832, "actual_norm": 9.437359809875488, "relative_norm": 0.04999897837419257, "target_actual_norm": 9.437359809875488, "match_error": 0.0, "scalar": 0.25074957310025076, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1327, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.04310671508617646, -0.004983112694220874], "logp": -0.04808982778039733, "cache_id": 1328, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43132209777832, "actual_norm": 9.437359809875488, "relative_norm": 0.04999897837419257, "target_actual_norm": 9.437359809875488, "match_error": 0.0, "scalar": 0.25074957310025076, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-12.668106715086177, -1.4801343566523446e-05], "logp": -12.668121516429743, "cache_id": 1329, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43132209777832, "actual_norm": 9.437359809875488, "relative_norm": 0.04999897837419257, "target_actual_norm": 9.437359809875488, "match_error": 0.0, "scalar": 0.25074957310025076, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.620031688649345, "first_log_odds": -12.625, "later_log_odds": 0.004968311350655341, "first_differing_log_odds": -12.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-j.npz", "array_sha256": "db813c8fb71756e57a2f04e7aea2a764889efb6f30108c43ba1811411181c9fe", "seconds": 0.34296116698533297, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457501696, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-opposite", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.427281379699707, "actual_norm": 9.437185287475586, "relative_norm": 0.04999805375736389, "target_actual_norm": 9.437359809875488, "match_error": -0.00017452239990234375, "scalar": 9.427279622090282, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1330, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.0863503029341312, -0.005034524301481867], "logp": -0.09138482723561307, "cache_id": 1331, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.427281379699707, "actual_norm": 9.437185287475586, "relative_norm": 0.04999805375736389, "target_actual_norm": 9.437359809875488, "match_error": -0.00017452239990234375, "scalar": 9.427279622090282, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-12.96135030293413, -2.3906377426472628e-05], "logp": -12.961374209311558, "cache_id": 1332, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.427281379699707, "actual_norm": 9.437185287475586, "relative_norm": 0.04999805375736389, "target_actual_norm": 9.437359809875488, "match_error": -0.00017452239990234375, "scalar": 9.427279622090282, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.869989382075945, "first_log_odds": -12.875, "later_log_odds": 0.005010617924055438, "first_differing_log_odds": -12.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-opposite.npz", "array_sha256": "25c7321985a56bbac55cc5d3e496604c525948907232949dd7a05a84c9c0b3c2", "seconds": 0.3447032910771668, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457501696, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-unrelated", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.4382963180542, "actual_norm": 9.437458992004395, "relative_norm": 0.04999950383949412, "target_actual_norm": 9.437359809875488, "match_error": 9.918212890625e-05, "scalar": 9.438295827312686, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1333, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.04252087084357156, -0.004786445259962612], "logp": -0.047307316103534176, "cache_id": 1334, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.4382963180542, "actual_norm": 9.437458992004395, "relative_norm": 0.04999950383949412, "target_actual_norm": 9.437359809875488, "match_error": 9.918212890625e-05, "scalar": 9.438295827312686, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-13.167520870843571, -1.863703585503098e-05], "logp": -13.167539507879427, "cache_id": 1335, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.4382963180542, "actual_norm": 9.437458992004395, "relative_norm": 0.04999950383949412, "target_actual_norm": 9.437359809875488, "match_error": 9.918212890625e-05, "scalar": 9.438295827312686, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.120232191775893, "first_log_odds": -13.125, "later_log_odds": 0.004767808224107384, "first_differing_log_odds": -13.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-unrelated.npz", "array_sha256": "965bf4995aa3672104f2063eabd281b566c5e348a7d0abe851efb1241be454e7", "seconds": 0.3432582910172641, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457501696, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-ordinary", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.431464195251465, "actual_norm": 9.437283515930176, "relative_norm": 0.04999857417011446, "target_actual_norm": 9.437359809875488, "match_error": -7.62939453125e-05, "scalar": 9.431464700054676, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1336, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.06047781557792311, -0.00516071175473825], "logp": -0.06563852733266136, "cache_id": 1337, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.431464195251465, "actual_norm": 9.437283515930176, "relative_norm": 0.04999857417011446, "target_actual_norm": 9.437359809875488, "match_error": -7.62939453125e-05, "scalar": 9.431464700054676, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-12.810477815577924, -2.1016472808894128e-05], "logp": -12.810498832050733, "cache_id": 1338, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.431464195251465, "actual_norm": 9.437283515930176, "relative_norm": 0.04999857417011446, "target_actual_norm": 9.437359809875488, "match_error": -7.62939453125e-05, "scalar": 9.431464700054676, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.744860304718072, "first_log_odds": -12.75, "later_log_odds": 0.005139695281927814, "first_differing_log_odds": -12.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-ordinary.npz", "array_sha256": "a947dd524a60a5f38ffc06509b2856c75294b12ebfc50d3ea53a12807c95f72d", "seconds": 0.3431414160877466, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457534464, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-random0", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.42751407623291, "actual_norm": 9.437080383300781, "relative_norm": 0.04999749797675649, "target_actual_norm": 9.437359809875488, "match_error": -0.00027942657470703125, "scalar": 9.427513626449581, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1339, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.053955032049028316, -0.006252998752528589], "logp": -0.06020803080155691, "cache_id": 1340, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.42751407623291, "actual_norm": 9.437080383300781, "relative_norm": 0.04999749797675649, "target_actual_norm": 9.437359809875488, "match_error": -0.00027942657470703125, "scalar": 9.427513626449581, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-12.928955032049029, -1.9035895081499245e-05], "logp": -12.92897406794411, "cache_id": 1341, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.42751407623291, "actual_norm": 9.437080383300781, "relative_norm": 0.04999749797675649, "target_actual_norm": 9.437359809875488, "match_error": -0.00027942657470703125, "scalar": 9.427513626449581, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.868766037142553, "first_log_odds": -12.875, "later_log_odds": 0.00623396285744704, "first_differing_log_odds": -12.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-random0.npz", "array_sha256": "17d8e4a77604c4898865d2675acbc265e2235e7b9c1c3f3780a3813fccd3426d", "seconds": 0.3423483748920262, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457534464, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-random1", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.420007705688477, "actual_norm": 9.43740463256836, "relative_norm": 0.049999215844088465, "target_actual_norm": 9.437359809875488, "match_error": 4.482269287109375e-05, "scalar": 9.420007486616669, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1342, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.037785330980769284, -0.006352614470226157], "logp": -0.04413794545099544, "cache_id": 1343, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.420007705688477, "actual_norm": 9.43740463256836, "relative_norm": 0.049999215844088465, "target_actual_norm": 9.437359809875488, "match_error": 4.482269287109375e-05, "scalar": 9.420007486616669, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-13.28778533098077, -1.845358279989734e-05], "logp": -13.28780378456357, "cache_id": 1344, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.420007705688477, "actual_norm": 9.43740463256836, "relative_norm": 0.049999215844088465, "target_actual_norm": 9.437359809875488, "match_error": 4.482269287109375e-05, "scalar": 9.420007486616669, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.243665839112575, "first_log_odds": -13.25, "later_log_odds": 0.006334160887425, "first_differing_log_odds": -13.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-random1.npz", "array_sha256": "7b24650e176be401a3086c8d97cc7033c95fee4fb6caa995300c54c6a67ad398", "seconds": 0.3377246670424938, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457534464, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-random2", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "ILS<|im_end|>", "output_text": "ILS", "output_ids": [1715, 50, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43469524383545, "actual_norm": 9.43730354309082, "relative_norm": 0.04999868027368497, "target_actual_norm": 9.437359809875488, "match_error": -5.626678466796875e-05, "scalar": 9.43469576024654, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1345, "cached": true, "scoring": {"normalized": "ILS", "bare_format": true, "malformed": false, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-0.04772761901234653, -0.006763203596764211], "logp": -0.05449082260911074, "cache_id": 1346, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43469524383545, "actual_norm": 9.43730354309082, "relative_norm": 0.04999868027368497, "target_actual_norm": 9.437359809875488, "match_error": -5.626678466796875e-05, "scalar": 9.43469576024654, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-12.922727619012347, -1.897967303799995e-05], "logp": -12.922746598685386, "cache_id": 1347, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 188.7510528564453, "strength_one_norm": 37.61250686645508, "target_shrink_scale": 0.25074957310025076, "requested_norm": 9.43469524383545, "actual_norm": 9.43730354309082, "relative_norm": 0.04999868027368497, "target_actual_norm": 9.437359809875488, "match_error": -5.626678466796875e-05, "scalar": 9.43469576024654, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -12.868255776076275, "first_log_odds": -12.875, "later_log_odds": 0.006744223923725201, "first_differing_log_odds": -12.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-random2.npz", "array_sha256": "051e9d952545a29bb0e408616f1762a9696c6ad0a658f0d620a87d48619ee7e7", "seconds": 0.34868604224175215, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457534464, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-main-bridge", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 212.25390625, 359.33929443359375, 559.687744140625, 39.826332092285156, 158.55499267578125, 21.459884643554688, 22.431028366088867, 79.93527221679688, 62.151268005371094, 15.757245063781738, 155.8780059814453, 98.72683715820312, 85.53571319580078, 53.589332580566406, 194.9681396484375, 21.36212921142578, 155.17068481445312, 23.352476119995117, 67.08102416992188, 76.10604095458984, 12.170317649841309, 292.7027893066406, 83.72186279296875, 216.14340209960938, 351.85736083984375, 254.7342529296875, 192.0415802001953, 544.546875]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 212.30776977539062, 359.3491516113281, 559.6318359375, 39.85857391357422, 158.5707244873047, 21.50640296936035, 22.48086929321289, 79.92716979980469, 62.15192413330078, 15.792614936828613, 155.907958984375, 98.81393432617188, 85.57987976074219, 53.544612884521484, 194.92837524414062, 21.435649871826172, 155.1433868408203, 23.405391693115234, 67.10650634765625, 76.15579223632812, 12.18594741821289, 292.736083984375, 83.76739501953125, 216.1631317138672, 351.9225769042969, 254.68557739257812, 192.13302612304688, 544.6209716796875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15729913115501404, 0.26962465047836304, 0.5467166304588318, 0.04165329784154892, 0.1578751653432846, 0.020300578325986862, 0.022008610889315605, 0.0785192921757698, 0.05513126030564308, 0.015675926581025124, 0.14914141595363617, 0.10100472718477249, 0.0892760157585144, 0.05436280742287636, 0.18786044418811798, 0.020372923463582993, 0.12557779252529144, 0.023453140631318092, 0.07431655377149582, 0.08472108095884323, 0.014615381136536598, 0.2923940420150757, 0.08863265812397003, 0.1906166672706604, 0.3414436876773834, 0.27491316199302673, 0.18515172600746155, 0.5061427354812622]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1348, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-15.64044400198061, -0.006909218314333965], "logp": -15.647353220294944, "cache_id": 1349, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 212.25390625, 359.33929443359375, 559.687744140625, 39.826332092285156, 158.55499267578125, 21.459884643554688, 22.431028366088867, 79.93527221679688, 62.151268005371094, 15.757245063781738, 155.8780059814453, 98.72683715820312, 85.53571319580078, 53.589332580566406, 194.9681396484375, 21.36212921142578, 155.17068481445312, 23.352476119995117, 67.08102416992188, 76.10604095458984, 12.170317649841309, 292.7027893066406, 83.72186279296875, 216.14340209960938, 351.85736083984375, 254.7342529296875, 192.0415802001953, 544.546875]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 212.30776977539062, 359.3491516113281, 559.6318359375, 39.85857391357422, 158.5707244873047, 21.50640296936035, 22.48086929321289, 79.92716979980469, 62.15192413330078, 15.792614936828613, 155.907958984375, 98.81393432617188, 85.57987976074219, 53.544612884521484, 194.92837524414062, 21.435649871826172, 155.1433868408203, 23.405391693115234, 67.10650634765625, 76.15579223632812, 12.18594741821289, 292.736083984375, 83.76739501953125, 216.1631317138672, 351.9225769042969, 254.68557739257812, 192.13302612304688, 544.6209716796875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15729913115501404, 0.26962465047836304, 0.5467166304588318, 0.04165329784154892, 0.1578751653432846, 0.020300578325986862, 0.022008610889315605, 0.0785192921757698, 0.05513126030564308, 0.015675926581025124, 0.14914141595363617, 0.10100472718477249, 0.0892760157585144, 0.05436280742287636, 0.18786044418811798, 0.020372923463582993, 0.12557779252529144, 0.023453140631318092, 0.07431655377149582, 0.08472108095884323, 0.014615381136536598, 0.2923940420150757, 0.08863265812397003, 0.1906166672706604, 0.3414436876773834, 0.27491316199302673, 0.18515172600746155, 0.5061427354812622]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.015444001980610179, -9.795918015323383e-06], "logp": -0.015453797898625502, "cache_id": 1350, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 212.25390625, 359.33929443359375, 559.687744140625, 39.826332092285156, 158.55499267578125, 21.459884643554688, 22.431028366088867, 79.93527221679688, 62.151268005371094, 15.757245063781738, 155.8780059814453, 98.72683715820312, 85.53571319580078, 53.589332580566406, 194.9681396484375, 21.36212921142578, 155.17068481445312, 23.352476119995117, 67.08102416992188, 76.10604095458984, 12.170317649841309, 292.7027893066406, 83.72186279296875, 216.14340209960938, 351.85736083984375, 254.7342529296875, 192.0415802001953, 544.546875]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 212.30776977539062, 359.3491516113281, 559.6318359375, 39.85857391357422, 158.5707244873047, 21.50640296936035, 22.48086929321289, 79.92716979980469, 62.15192413330078, 15.792614936828613, 155.907958984375, 98.81393432617188, 85.57987976074219, 53.544612884521484, 194.92837524414062, 21.435649871826172, 155.1433868408203, 23.405391693115234, 67.10650634765625, 76.15579223632812, 12.18594741821289, 292.736083984375, 83.76739501953125, 216.1631317138672, 351.9225769042969, 254.68557739257812, 192.13302612304688, 544.6209716796875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15729913115501404, 0.26962465047836304, 0.5467166304588318, 0.04165329784154892, 0.1578751653432846, 0.020300578325986862, 0.022008610889315605, 0.0785192921757698, 0.05513126030564308, 0.015675926581025124, 0.14914141595363617, 0.10100472718477249, 0.0892760157585144, 0.05436280742287636, 0.18786044418811798, 0.020372923463582993, 0.12557779252529144, 0.023453140631318092, 0.07431655377149582, 0.08472108095884323, 0.014615381136536598, 0.2923940420150757, 0.08863265812397003, 0.1906166672706604, 0.3414436876773834, 0.27491316199302673, 0.18515172600746155, 0.5061427354812622]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.631899422396318, "first_log_odds": 15.625, "later_log_odds": 0.006899422396317689, "first_differing_log_odds": 15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-main-bridge.npz", "array_sha256": "4837f5bb0b4751547e56ed79485c0c828b5be4a20ee5de89de1d8730f83fe679", "seconds": 0.3332619578577578, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-baseline", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1351, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-24.500000035683335, -0.15119540117643565], "logp": -24.651195436859773, "cache_id": 1352, "trace": []}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.000000035683335, -5.599182249521647e-05], "logp": -30.00005602750583, "cache_id": 1353, "trace": []}, "log_odds": -5.3488605906460585, "first_log_odds": -5.5, "later_log_odds": 0.15113940935394155, "first_differing_log_odds": -5.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-baseline.npz", "array_sha256": "60b292e936157fe34bebb76ed93065c364536ddf8789f860536ea1e785e61c3a", "seconds": 1.236695250030607, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-noop", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.42482852935791, "match_error": -9.42482852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1354, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-24.500000035683335, -0.15119540117643565], "logp": -24.651195436859773, "cache_id": 1355, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.42482852935791, "match_error": -9.42482852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.000000035683335, -5.599182249521647e-05], "logp": -30.00005602750583, "cache_id": 1356, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.42482852935791, "match_error": -9.42482852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.3488605906460585, "first_log_odds": -5.5, "later_log_odds": 0.15113940935394155, "first_differing_log_odds": -5.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-noop.npz", "array_sha256": "bd8adb03511af45da4648878459f9eae1b46656ecbff6992541e964054e106a2", "seconds": 1.2597384173423052, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-j", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.405808448791504, "actual_norm": 9.42482852935791, "relative_norm": 0.04999820398737064, "target_actual_norm": 9.42482852935791, "match_error": 0.0, "scalar": 0.2500713616089584, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1357, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-26.125000017289217, -0.12965089489131496], "logp": -26.254650912180534, "cache_id": 1358, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.405808448791504, "actual_norm": 9.42482852935791, "relative_norm": 0.04999820398737064, "target_actual_norm": 9.42482852935791, "match_error": 0.0, "scalar": 0.2500713616089584, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.000000017289217, -3.7456546977483016e-05], "logp": -30.000037473836194, "cache_id": 1359, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.405808448791504, "actual_norm": 9.42482852935791, "relative_norm": 0.04999820398737064, "target_actual_norm": 9.42482852935791, "match_error": 0.0, "scalar": 0.2500713616089584, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -3.7453865616556605, "first_log_odds": -3.875, "later_log_odds": 0.12961343834433947, "first_differing_log_odds": -3.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-j.npz", "array_sha256": "086e79b5ce7d860c7b4bfe950e81f14d388ac50b84243259639e731521b4f5ff", "seconds": 1.254706374835223, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-opposite", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.41037654876709, "actual_norm": 9.424757957458496, "relative_norm": 0.04999782960727325, "target_actual_norm": 9.42482852935791, "match_error": -7.05718994140625e-05, "scalar": 9.4103754765747, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1360, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.000000026926553, -0.14680818680206426], "logp": -25.14680821372862, "cache_id": 1361, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.41037654876709, "actual_norm": 9.424757957458496, "relative_norm": 0.04999782960727325, "target_actual_norm": 9.42482852935791, "match_error": -7.05718994140625e-05, "scalar": 9.4103754765747, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.937500026926553, -8.138112871951962e-05], "logp": -30.937581408055273, "cache_id": 1362, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.41037654876709, "actual_norm": 9.424757957458496, "relative_norm": 0.04999782960727325, "target_actual_norm": 9.42482852935791, "match_error": -7.05718994140625e-05, "scalar": 9.4103754765747, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.790773194326654, "first_log_odds": -5.9375, "later_log_odds": 0.14672680567334595, "first_differing_log_odds": -5.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-opposite.npz", "array_sha256": "1e6a23fbe1639c282e5181726545bc3adf72b746c667b17bda8fbd4ec61a01bc", "seconds": 1.2339215409010649, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-unrelated", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.43712329864502, "actual_norm": 9.424739837646484, "relative_norm": 0.049997733482653656, "target_actual_norm": 9.42482852935791, "match_error": -8.869171142578125e-05, "scalar": 9.437124410083925, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1363, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.62500001729044, -0.16110156670563985], "logp": -25.78610158399608, "cache_id": 1364, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.43712329864502, "actual_norm": 9.424739837646484, "relative_norm": 0.049997733482653656, "target_actual_norm": 9.42482852935791, "match_error": -8.869171142578125e-05, "scalar": 9.437124410083925, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-31.18750001729044, -5.566040103689422e-05], "logp": -31.187555677691478, "cache_id": 1365, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.43712329864502, "actual_norm": 9.424739837646484, "relative_norm": 0.049997733482653656, "target_actual_norm": 9.42482852935791, "match_error": -8.869171142578125e-05, "scalar": 9.437124410083925, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.401454093695399, "first_log_odds": -5.5625, "later_log_odds": 0.1610459063046008, "first_differing_log_odds": -5.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-unrelated.npz", "array_sha256": "f9d4e13c46897d123a27dd5110ce7fefa297f6d85a09ac26e32877fe9a249b8f", "seconds": 1.2518919580616057, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-ordinary", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.421089172363281, "actual_norm": 9.425126075744629, "relative_norm": 0.049999782454808274, "target_actual_norm": 9.42482852935791, "match_error": 0.00029754638671875, "scalar": 9.421089431125438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1366, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.37500003327926, -0.15579970753634836], "logp": -25.53079974081561, "cache_id": 1367, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.421089172363281, "actual_norm": 9.425126075744629, "relative_norm": 0.049999782454808274, "target_actual_norm": 9.42482852935791, "match_error": 0.00029754638671875, "scalar": 9.421089431125438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-29.87500003327926, -5.446422836340778e-05], "logp": -29.875054497507623, "cache_id": 1368, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.421089172363281, "actual_norm": 9.425126075744629, "relative_norm": 0.049999782454808274, "target_actual_norm": 9.42482852935791, "match_error": 0.00029754638671875, "scalar": 9.421089431125438, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -4.344254756692013, "first_log_odds": -4.5, "later_log_odds": 0.15574524330798667, "first_differing_log_odds": -4.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-ordinary.npz", "array_sha256": "a831bf07820ee3a8d1c90a5251a48aaee2daf1b109f053fd1e220016b67f7c0f", "seconds": 1.2379777077585459, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-random0", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.422438621520996, "actual_norm": 9.425034523010254, "relative_norm": 0.04999929677251977, "target_actual_norm": 9.42482852935791, "match_error": 0.00020599365234375, "scalar": 9.422437663661185, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1369, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.125000028050724, -0.13773952964157046], "logp": -25.262739557692296, "cache_id": 1370, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.422438621520996, "actual_norm": 9.425034523010254, "relative_norm": 0.04999929677251977, "target_actual_norm": 9.42482852935791, "match_error": 0.00020599365234375, "scalar": 9.422437663661185, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.312500028050724, -5.628629336989526e-05], "logp": -30.312556314344093, "cache_id": 1371, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.422438621520996, "actual_norm": 9.425034523010254, "relative_norm": 0.04999929677251977, "target_actual_norm": 9.42482852935791, "match_error": 0.00020599365234375, "scalar": 9.422437663661185, "random_seed": 20273208, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.049816756651797, "first_log_odds": -5.1875, "later_log_odds": 0.13768324334820292, "first_differing_log_odds": -5.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-random0.npz", "array_sha256": "a9b0a2924e2b28c1e2030482fba313123611ffc0dc5d4856686517c72f4a6eef", "seconds": 1.3093043752014637, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-random1", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.412551879882812, "actual_norm": 9.424765586853027, "relative_norm": 0.0499978700807973, "target_actual_norm": 9.42482852935791, "match_error": -6.29425048828125e-05, "scalar": 9.412550625065705, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1372, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.875000023802567, -0.17297947315648685], "logp": -26.047979496959055, "cache_id": 1373, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.412551879882812, "actual_norm": 9.424765586853027, "relative_norm": 0.0499978700807973, "target_actual_norm": 9.42482852935791, "match_error": -6.29425048828125e-05, "scalar": 9.412550625065705, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.562500023802567, -6.222581239807243e-05], "logp": -30.562562249614967, "cache_id": 1374, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.412551879882812, "actual_norm": 9.424765586853027, "relative_norm": 0.0499978700807973, "target_actual_norm": 9.42482852935791, "match_error": -6.29425048828125e-05, "scalar": 9.412550625065705, "random_seed": 20273209, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -4.514582752655912, "first_log_odds": -4.6875, "later_log_odds": 0.1729172473440883, "first_differing_log_odds": -4.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-random1.npz", "array_sha256": "e488a27bc95afabd7d74f6cf4b8119d6a199e4044a280698f1e3b26034f036b0", "seconds": 1.2685213750228286, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-random2", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.419795036315918, "actual_norm": 9.42483901977539, "relative_norm": 0.0499982596384662, "target_actual_norm": 9.42482852935791, "match_error": 1.049041748046875e-05, "scalar": 9.41979512789112, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1375, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-25.500000022709955, -0.14829605543031044], "logp": -25.648296078140266, "cache_id": 1376, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.419795036315918, "actual_norm": 9.42483901977539, "relative_norm": 0.0499982596384662, "target_actual_norm": 9.42482852935791, "match_error": 1.049041748046875e-05, "scalar": 9.41979512789112, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-30.625000022709955, -5.5347212419624564e-05], "logp": -30.625055369922375, "cache_id": 1377, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 188.5033416748047, "strength_one_norm": 37.61249542236328, "target_shrink_scale": 0.2500713616089584, "requested_norm": 9.419795036315918, "actual_norm": 9.42483901977539, "relative_norm": 0.0499982596384662, "target_actual_norm": 9.42482852935791, "match_error": 1.049041748046875e-05, "scalar": 9.41979512789112, "random_seed": 20273210, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -4.976759291782109, "first_log_odds": -5.125, "later_log_odds": 0.14824070821789093, "first_differing_log_odds": -5.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-random2.npz", "array_sha256": "da093b96ab6ead1ad642a85fdc232766d958fdfbe62c54a66ab1016be75681b2", "seconds": 1.2673040837980807, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-23-paraphrase-bridge", "case_id": "eval-23", "city": "Ashdod", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).<|im_end|>", "output_text": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "output_ids": [785, 11413, 1483, 304, 14595, 67, 347, 374, 279, 15907, 2932, 17659, 320, 40, 808, 8382, 13, 220, 19, 17, 16, 22, 2038, 25, 3070, 1715, 50, 334, 568, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 179.74256896972656, 219.32681274414062, 606.0149536132812, 88.59004974365234, 21.482690811157227, 226.4809112548828, 109.99984741210938, 65.54439544677734, 54.449893951416016, 6.4662699699401855, 163.90953063964844, 96.72039794921875, 95.39094543457031, 51.127105712890625, 183.51315307617188, 99.89681243896484, 107.55799865722656, 46.86962890625, 288.60540771484375, 111.89684295654297, 217.86923217773438, 362.91375732421875, 299.5835266113281, 196.00889587402344, 122.91168975830078]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 179.69247436523438, 219.32275390625, 606.0553588867188, 88.5896224975586, 21.57727813720703, 226.4620361328125, 109.99768829345703, 65.60993194580078, 54.46809387207031, 6.663527488708496, 163.9494171142578, 96.6950912475586, 95.42961883544922, 51.14400100708008, 183.4899139404297, 99.90311431884766, 107.57815551757812, 46.84591293334961, 288.6181640625, 111.83007049560547, 217.8870849609375, 362.8965759277344, 299.541259765625, 196.0093536376953, 122.92497253417969]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.1336858719587326, 0.16601063311100006, 0.5858189463615417, 0.08847236633300781, 0.019777007400989532, 0.22009213268756866, 0.10062268376350403, 0.06448210030794144, 0.0469464436173439, 0.00633541913703084, 0.15461677312850952, 0.09781955927610397, 0.09810557216405869, 0.050056517124176025, 0.1730114221572876, 0.10577812045812607, 0.12152190506458282, 0.05382801219820976, 0.2839152216911316, 0.11832547932863235, 0.19221848249435425, 0.3470304012298584, 0.31362268328666687, 0.18846829235553741, 0.1293015033006668]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1378, "cached": true, "scoring": {"normalized": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **ILS**).", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ashdod is the Israeli Shekel (I.S.O. 4217 code: **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 35, "end": 42}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.750000121087737, -0.22505073038798015], "logp": -27.975050851475718, "cache_id": 1379, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 179.74256896972656, 219.32681274414062, 606.0149536132812, 88.59004974365234, 21.482690811157227, 226.4809112548828, 109.99984741210938, 65.54439544677734, 54.449893951416016, 6.4662699699401855, 163.90953063964844, 96.72039794921875, 95.39094543457031, 51.127105712890625, 183.51315307617188, 99.89681243896484, 107.55799865722656, 46.86962890625, 288.60540771484375, 111.89684295654297, 217.86923217773438, 362.91375732421875, 299.5835266113281, 196.00889587402344, 122.91168975830078]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 179.69247436523438, 219.32275390625, 606.0553588867188, 88.5896224975586, 21.57727813720703, 226.4620361328125, 109.99768829345703, 65.60993194580078, 54.46809387207031, 6.663527488708496, 163.9494171142578, 96.6950912475586, 95.42961883544922, 51.14400100708008, 183.4899139404297, 99.90311431884766, 107.57815551757812, 46.84591293334961, 288.6181640625, 111.83007049560547, 217.8870849609375, 362.8965759277344, 299.541259765625, 196.0093536376953, 122.92497253417969]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.1336858719587326, 0.16601063311100006, 0.5858189463615417, 0.08847236633300781, 0.019777007400989532, 0.22009213268756866, 0.10062268376350403, 0.06448210030794144, 0.0469464436173439, 0.00633541913703084, 0.15461677312850952, 0.09781955927610397, 0.09810557216405869, 0.050056517124176025, 0.1730114221572876, 0.10577812045812607, 0.12152190506458282, 0.05382801219820976, 0.2839152216911316, 0.11832547932863235, 0.19221848249435425, 0.3470304012298584, 0.31362268328666687, 0.18846829235553741, 0.1293015033006668]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.750000121087737, -3.7794571218871924e-05], "logp": -22.750037915658957, "cache_id": 1380, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 179.74256896972656, 219.32681274414062, 606.0149536132812, 88.59004974365234, 21.482690811157227, 226.4809112548828, 109.99984741210938, 65.54439544677734, 54.449893951416016, 6.4662699699401855, 163.90953063964844, 96.72039794921875, 95.39094543457031, 51.127105712890625, 183.51315307617188, 99.89681243896484, 107.55799865722656, 46.86962890625, 288.60540771484375, 111.89684295654297, 217.86923217773438, 362.91375732421875, 299.5835266113281, 196.00889587402344, 122.91168975830078]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 179.69247436523438, 219.32275390625, 606.0553588867188, 88.5896224975586, 21.57727813720703, 226.4620361328125, 109.99768829345703, 65.60993194580078, 54.46809387207031, 6.663527488708496, 163.9494171142578, 96.6950912475586, 95.42961883544922, 51.14400100708008, 183.4899139404297, 99.90311431884766, 107.57815551757812, 46.84591293334961, 288.6181640625, 111.83007049560547, 217.8870849609375, 362.8965759277344, 299.541259765625, 196.0093536376953, 122.92497253417969]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.1336858719587326, 0.16601063311100006, 0.5858189463615417, 0.08847236633300781, 0.019777007400989532, 0.22009213268756866, 0.10062268376350403, 0.06448210030794144, 0.0469464436173439, 0.00633541913703084, 0.15461677312850952, 0.09781955927610397, 0.09810557216405869, 0.050056517124176025, 0.1730114221572876, 0.10577812045812607, 0.12152190506458282, 0.05382801219820976, 0.2839152216911316, 0.11832547932863235, 0.19221848249435425, 0.3470304012298584, 0.31362268328666687, 0.18846829235553741, 0.1293015033006668]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 5.2250129358167605, "first_log_odds": 5.0, "later_log_odds": 0.22501293581676052, "first_differing_log_odds": 5.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-23-paraphrase-bridge.npz", "array_sha256": "7182d4173143a693a4e2b781d7cb2e9e821211b2833505eafd67222cd05d670d", "seconds": 1.2523207091726363, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-baseline", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1381, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.801293947108699, -0.0022588141512859747], "logp": -7.8035527612599855, "cache_id": 1382, "trace": []}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-9.0512939471087, -2.7408089048107175e-05], "logp": -9.051321355197748, "cache_id": 1383, "trace": []}, "log_odds": -1.2477685939377627, "first_log_odds": -1.2500000000000009, "later_log_odds": 0.0022314060622381504, "first_differing_log_odds": -1.2500000000000009, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-baseline.npz", "array_sha256": "bc3e3534906d5e2cfd39c5e4278bb1045b9aa48dd4da7bdedca9f987248034d9", "seconds": 0.3436574172228575, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-noop", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.76857852935791, "match_error": -9.76857852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1384, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.801293947108699, -0.0022588141512859747], "logp": -7.8035527612599855, "cache_id": 1385, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.76857852935791, "match_error": -9.76857852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-9.0512939471087, -2.7408089048107175e-05], "logp": -9.051321355197748, "cache_id": 1386, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.76857852935791, "match_error": -9.76857852935791, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.2477685939377627, "first_log_odds": -1.2500000000000009, "later_log_odds": 0.0022314060622381504, "first_differing_log_odds": -1.2500000000000009, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-noop.npz", "array_sha256": "067a9ee015bde09f09a249f6d107dd4d1025857bbcf26af5273ffa7fe75781df", "seconds": 0.3195009999908507, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-j", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.76272964477539, "actual_norm": 9.76857852935791, "relative_norm": 0.049999387887936515, "target_actual_norm": 9.76857852935791, "match_error": 0.0, "scalar": 0.2758767157610894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1387, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-8.982373055660528, -0.0018686863701616982], "logp": -8.98424174203069, "cache_id": 1388, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.76272964477539, "actual_norm": 9.76857852935791, "relative_norm": 0.049999387887936515, "target_actual_norm": 9.76857852935791, "match_error": 0.0, "scalar": 0.2758767157610894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.107373055660528, -1.8342637316824455e-05], "logp": -8.107391398297844, "cache_id": 1389, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.76272964477539, "actual_norm": 9.76857852935791, "relative_norm": 0.049999387887936515, "target_actual_norm": 9.76857852935791, "match_error": 0.0, "scalar": 0.2758767157610894, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.8768503437328459, "first_log_odds": 0.875, "later_log_odds": 0.0018503437328458716, "first_differing_log_odds": 0.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-j.npz", "array_sha256": "614e05beb4fa34b5e9a87d512b9dbe9228bedd50d2538755657a8015e8869fc2", "seconds": 0.3193732500076294, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-opposite", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.764701843261719, "actual_norm": 9.768611907958984, "relative_norm": 0.04999955873261133, "target_actual_norm": 9.76857852935791, "match_error": 3.337860107421875e-05, "scalar": 9.764703055698192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1390, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.092614272782175, -0.0027875642648126737], "logp": -7.095401837046987, "cache_id": 1391, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.764701843261719, "actual_norm": 9.768611907958984, "relative_norm": 0.04999955873261133, "target_actual_norm": 9.76857852935791, "match_error": 3.337860107421875e-05, "scalar": 9.764703055698192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-9.467614272782175, -2.7546118055274547e-05], "logp": -9.46764181890023, "cache_id": 1392, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.764701843261719, "actual_norm": 9.768611907958984, "relative_norm": 0.04999955873261133, "target_actual_norm": 9.76857852935791, "match_error": 3.337860107421875e-05, "scalar": 9.764703055698192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -2.3722399818532427, "first_log_odds": -2.375, "later_log_odds": 0.0027600181467573037, "first_differing_log_odds": -2.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-opposite.npz", "array_sha256": "243221ce8d3ed169ae0e14b64ca79e873d76dff6c50c02126ffed2936d03888a", "seconds": 0.31979270884767175, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-unrelated", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.775321960449219, "actual_norm": 9.76866340637207, "relative_norm": 0.04999982232153819, "target_actual_norm": 9.76857852935791, "match_error": 8.487701416015625e-05, "scalar": 9.775323344092612, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1393, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-8.303952160748914, -0.0032749342437100957], "logp": -8.307227094992623, "cache_id": 1394, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.775321960449219, "actual_norm": 9.76866340637207, "relative_norm": 0.04999982232153819, "target_actual_norm": 9.76857852935791, "match_error": 8.487701416015625e-05, "scalar": 9.775323344092612, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-9.178952160748914, -2.5644039520775223e-05], "logp": -9.178977804788435, "cache_id": 1395, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.775321960449219, "actual_norm": 9.76866340637207, "relative_norm": 0.04999982232153819, "target_actual_norm": 9.76857852935791, "match_error": 8.487701416015625e-05, "scalar": 9.775323344092612, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.8717507097958119, "first_log_odds": -0.875, "later_log_odds": 0.003249290204188071, "first_differing_log_odds": -0.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-unrelated.npz", "array_sha256": "06b3136d00c3e039139ecadd62814be8d9ed0d92ae25bb34b1c1ff0f58c5ec47", "seconds": 0.32393245911225677, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-ordinary", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.771650314331055, "actual_norm": 9.768575668334961, "relative_norm": 0.04999937324410725, "target_actual_norm": 9.76857852935791, "match_error": -2.86102294921875e-06, "scalar": 9.771652823366821, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1396, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-8.126966960852837, -0.002023093888698549], "logp": -8.128990054741536, "cache_id": 1397, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.771650314331055, "actual_norm": 9.768575668334961, "relative_norm": 0.04999937324410725, "target_actual_norm": 9.76857852935791, "match_error": -2.86102294921875e-06, "scalar": 9.771652823366821, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.876966960852837, -2.3134297561416976e-05], "logp": -8.876990095150399, "cache_id": 1398, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.771650314331055, "actual_norm": 9.768575668334961, "relative_norm": 0.04999937324410725, "target_actual_norm": 9.76857852935791, "match_error": -2.86102294921875e-06, "scalar": 9.771652823366821, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.7480000404088631, "first_log_odds": -0.75, "later_log_odds": 0.0019999595911368573, "first_differing_log_odds": -0.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-ordinary.npz", "array_sha256": "0482acc8b00b103326a12c8ce7cf2fb91b6d342b3fa966dbaf93d840a9e5ec01", "seconds": 0.32152154203504324, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-random0", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77161693572998, "actual_norm": 9.767107963562012, "relative_norm": 0.04999186095969175, "target_actual_norm": 9.76857852935791, "match_error": -0.0014705657958984375, "scalar": 9.771615559197016, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1399, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.882872804915488, -0.0025631801247596826], "logp": -7.885435985040248, "cache_id": 1400, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77161693572998, "actual_norm": 9.767107963562012, "relative_norm": 0.04999186095969175, "target_actual_norm": 9.76857852935791, "match_error": -0.0014705657958984375, "scalar": 9.771615559197016, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.882872804915488, -2.6488547298511518e-05], "logp": -8.882899293462787, "cache_id": 1401, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77161693572998, "actual_norm": 9.767107963562012, "relative_norm": 0.04999186095969175, "target_actual_norm": 9.76857852935791, "match_error": -0.0014705657958984375, "scalar": 9.771615559197016, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.9974633084225388, "first_log_odds": -1.0, "later_log_odds": 0.00253669157746117, "first_differing_log_odds": -1.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-random0.npz", "array_sha256": "4731984532ac449e92a4823c81de5d46865103613097ec5b31df672792915a36", "seconds": 0.32300887489691377, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-random1", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.772008895874023, "actual_norm": 9.768678665161133, "relative_norm": 0.049999900421960966, "target_actual_norm": 9.76857852935791, "match_error": 0.00010013580322265625, "scalar": 9.772006832979969, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1402, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-7.534363773216887, -0.0022649266294934023], "logp": -7.53662869984638, "cache_id": 1403, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.772008895874023, "actual_norm": 9.768678665161133, "relative_norm": 0.049999900421960966, "target_actual_norm": 9.76857852935791, "match_error": 0.00010013580322265625, "scalar": 9.772006832979969, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-9.034363773216887, -2.3464459261479877e-05], "logp": -9.034387237676148, "cache_id": 1404, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.772008895874023, "actual_norm": 9.768678665161133, "relative_norm": 0.049999900421960966, "target_actual_norm": 9.76857852935791, "match_error": 0.00010013580322265625, "scalar": 9.772006832979969, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.4977585378297675, "first_log_odds": -1.5, "later_log_odds": 0.0022414621702324666, "first_differing_log_odds": -1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-random1.npz", "array_sha256": "71ee6338a84315c302f70c3e1f1c150175be9de01ee8359e11c5b321bdef69cb", "seconds": 0.3197129578329623, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-random2", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77042293548584, "actual_norm": 9.768228530883789, "relative_norm": 0.04999759645948915, "target_actual_norm": 9.76857852935791, "match_error": -0.00034999847412109375, "scalar": 9.770423105763257, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1405, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-8.247600389212817, -0.0022702089195608772], "logp": -8.249870598132377, "cache_id": 1406, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77042293548584, "actual_norm": 9.768228530883789, "relative_norm": 0.04999759645948915, "target_actual_norm": 9.76857852935791, "match_error": -0.00034999847412109375, "scalar": 9.770423105763257, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.997600389212817, -2.2427844225962603e-05], "logp": -8.997622817057042, "cache_id": 1407, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 195.37396240234375, "strength_one_norm": 35.38801574707031, "target_shrink_scale": 0.2758767157610894, "requested_norm": 9.77042293548584, "actual_norm": 9.768228530883789, "relative_norm": 0.04999759645948915, "target_actual_norm": 9.76857852935791, "match_error": -0.00034999847412109375, "scalar": 9.770423105763257, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.7477522189246653, "first_log_odds": -0.75, "later_log_odds": 0.0022477810753347427, "first_differing_log_odds": -0.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-random2.npz", "array_sha256": "2779a0ed06a122ebbc446599098296591e5ac5aa230c4ad457d9cfac54fd3ea0", "seconds": 0.3186043342575431, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-main-bridge", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 173.50755310058594, 80.45114135742188, 502.8427734375, 52.651920318603516, 142.59645080566406, 6.01807975769043, 26.313871383666992, 96.64999389648438, 67.55584716796875, 18.696765899658203, 152.72914123535156, 103.65186309814453, 85.93645477294922, 54.7364616394043, 185.4855194091797, 32.85991668701172, 138.2173614501953, 15.900251388549805, 89.79342651367188, 81.20582580566406, 1.606684684753418, 285.86004638671875, 83.54464721679688, 206.579833984375, 347.4267578125, 260.7442321777344, 179.56776428222656, 617.943359375]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 173.55313110351562, 80.43064880371094, 502.8077087402344, 52.67277526855469, 142.60284423828125, 6.1457319259643555, 26.340877532958984, 96.60904693603516, 67.59186553955078, 18.752925872802734, 152.70758056640625, 103.64498138427734, 85.91791534423828, 54.69139099121094, 185.51242065429688, 32.8511962890625, 138.2459716796875, 15.917186737060547, 89.7961196899414, 81.18801879882812, 1.6277766227722168, 285.8682556152344, 83.58230590820312, 206.68487548828125, 347.39385986328125, 260.8031921386719, 179.60792541503906, 617.9051513671875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15281306207180023, 0.06052928790450096, 0.5004538297653198, 0.056632403284311295, 0.14294536411762238, 0.005841495469212532, 0.025827467441558838, 0.09455430507659912, 0.05938228592276573, 0.0185488760471344, 0.14654681086540222, 0.10600277781486511, 0.08944456279277802, 0.05545618757605553, 0.17832767963409424, 0.0312139131128788, 0.11184412240982056, 0.016057917848229408, 0.10141172260046005, 0.08974727988243103, 0.0019490823615342379, 0.28622373938560486, 0.08823265135288239, 0.18173439800739288, 0.33734044432640076, 0.282632440328598, 0.17346309125423431, 0.562518298625946]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1408, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-19.031333766171816, -0.0075847747427340625], "logp": -19.03891854091455, "cache_id": 1409, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 173.50755310058594, 80.45114135742188, 502.8427734375, 52.651920318603516, 142.59645080566406, 6.01807975769043, 26.313871383666992, 96.64999389648438, 67.55584716796875, 18.696765899658203, 152.72914123535156, 103.65186309814453, 85.93645477294922, 54.7364616394043, 185.4855194091797, 32.85991668701172, 138.2173614501953, 15.900251388549805, 89.79342651367188, 81.20582580566406, 1.606684684753418, 285.86004638671875, 83.54464721679688, 206.579833984375, 347.4267578125, 260.7442321777344, 179.56776428222656, 617.943359375]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 173.55313110351562, 80.43064880371094, 502.8077087402344, 52.67277526855469, 142.60284423828125, 6.1457319259643555, 26.340877532958984, 96.60904693603516, 67.59186553955078, 18.752925872802734, 152.70758056640625, 103.64498138427734, 85.91791534423828, 54.69139099121094, 185.51242065429688, 32.8511962890625, 138.2459716796875, 15.917186737060547, 89.7961196899414, 81.18801879882812, 1.6277766227722168, 285.8682556152344, 83.58230590820312, 206.68487548828125, 347.39385986328125, 260.8031921386719, 179.60792541503906, 617.9051513671875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15281306207180023, 0.06052928790450096, 0.5004538297653198, 0.056632403284311295, 0.14294536411762238, 0.005841495469212532, 0.025827467441558838, 0.09455430507659912, 0.05938228592276573, 0.0185488760471344, 0.14654681086540222, 0.10600277781486511, 0.08944456279277802, 0.05545618757605553, 0.17832767963409424, 0.0312139131128788, 0.11184412240982056, 0.016057917848229408, 0.10141172260046005, 0.08974727988243103, 0.0019490823615342379, 0.28622373938560486, 0.08823265135288239, 0.18173439800739288, 0.33734044432640076, 0.282632440328598, 0.17346309125423431, 0.562518298625946]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.03133376617181727, -1.6137055731644e-05], "logp": -0.03134990322754891, "cache_id": 1410, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 62.28446578979492, 126.84515380859375, 25.45504379272461, 146.898681640625, 149.28448486328125, 173.50755310058594, 80.45114135742188, 502.8427734375, 52.651920318603516, 142.59645080566406, 6.01807975769043, 26.313871383666992, 96.64999389648438, 67.55584716796875, 18.696765899658203, 152.72914123535156, 103.65186309814453, 85.93645477294922, 54.7364616394043, 185.4855194091797, 32.85991668701172, 138.2173614501953, 15.900251388549805, 89.79342651367188, 81.20582580566406, 1.606684684753418, 285.86004638671875, 83.54464721679688, 206.579833984375, 347.4267578125, 260.7442321777344, 179.56776428222656, 617.943359375]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 62.26139450073242, 126.80125427246094, 25.503572463989258, 146.92108154296875, 149.32481384277344, 173.55313110351562, 80.43064880371094, 502.8077087402344, 52.67277526855469, 142.60284423828125, 6.1457319259643555, 26.340877532958984, 96.60904693603516, 67.59186553955078, 18.752925872802734, 152.70758056640625, 103.64498138427734, 85.91791534423828, 54.69139099121094, 185.51242065429688, 32.8511962890625, 138.2459716796875, 15.917186737060547, 89.7961196899414, 81.18801879882812, 1.6277766227722168, 285.8682556152344, 83.58230590820312, 206.68487548828125, 347.39385986328125, 260.8031921386719, 179.60792541503906, 617.9051513671875]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.05715377256274223, 0.1112530529499054, 0.020748509094119072, 0.12575273215770721, 0.13158513605594635, 0.15281306207180023, 0.06052928790450096, 0.5004538297653198, 0.056632403284311295, 0.14294536411762238, 0.005841495469212532, 0.025827467441558838, 0.09455430507659912, 0.05938228592276573, 0.0185488760471344, 0.14654681086540222, 0.10600277781486511, 0.08944456279277802, 0.05545618757605553, 0.17832767963409424, 0.0312139131128788, 0.11184412240982056, 0.016057917848229408, 0.10141172260046005, 0.08974727988243103, 0.0019490823615342379, 0.28622373938560486, 0.08823265135288239, 0.18173439800739288, 0.33734044432640076, 0.282632440328598, 0.17346309125423431, 0.562518298625946]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 19.007568637687, "first_log_odds": 19.0, "later_log_odds": 0.007568637687001001, "first_differing_log_odds": 19.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-main-bridge.npz", "array_sha256": "d0d28f06b53c2fed4b3e16d8d75875e97af6d56c6d08843f1e6c7eac1ff5a60c", "seconds": 0.3453676658682525, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-baseline", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1411, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.875005766890638, -0.02680233869214625], "logp": -23.901808105582784, "cache_id": 1412, "trace": []}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-23.125005766890638, -5.823981152936425e-05], "logp": -23.125064006702168, "cache_id": 1413, "trace": []}, "log_odds": 0.7767440988806165, "first_log_odds": 0.75, "later_log_odds": 0.026744098880616463, "first_differing_log_odds": 0.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-baseline.npz", "array_sha256": "bb858dc2db93d5bbda9c4232900edf32275c137ae7aaf81a41c734a6160bc225", "seconds": 1.322140583768487, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-noop", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.726096153259277, "match_error": -9.726096153259277, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1414, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.875005766890638, -0.02680233869214625], "logp": -23.901808105582784, "cache_id": 1415, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.726096153259277, "match_error": -9.726096153259277, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-23.125005766890638, -5.823981152936425e-05], "logp": -23.125064006702168, "cache_id": 1416, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.726096153259277, "match_error": -9.726096153259277, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.7767440988806165, "first_log_odds": 0.75, "later_log_odds": 0.026744098880616463, "first_differing_log_odds": 0.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-noop.npz", "array_sha256": "6607c4d7863c9fbbf5d7b53b27bffeb189e3f20abd7f5c38a90e7186b7f40b2b", "seconds": 1.305036584381014, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-j", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727829933166504, "actual_norm": 9.726096153259277, "relative_norm": 0.04999110522077758, "target_actual_norm": 9.726096153259277, "match_error": 0.0, "scalar": 0.26394169229828157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1417, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-24.500009014251972, -0.023948339923286502], "logp": -24.52395735417526, "cache_id": 1418, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727829933166504, "actual_norm": 9.726096153259277, "relative_norm": 0.04999110522077758, "target_actual_norm": 9.726096153259277, "match_error": 0.0, "scalar": 0.26394169229828157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.000009014251972, -3.4648313490632634e-05], "logp": -22.000043662565464, "cache_id": 1419, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727829933166504, "actual_norm": 9.726096153259277, "relative_norm": 0.04999110522077758, "target_actual_norm": 9.726096153259277, "match_error": 0.0, "scalar": 0.26394169229828157, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 2.5239136916097955, "first_log_odds": 2.5, "later_log_odds": 0.023913691609795507, "first_differing_log_odds": 2.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-j.npz", "array_sha256": "41f7c15ea8a7a477111db9c6078862c4af4770e75e7bdafc5fb36c1a4643735c", "seconds": 1.3048893339000642, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-opposite", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.721794128417969, "actual_norm": 9.726202011108398, "relative_norm": 0.04999164931892245, "target_actual_norm": 9.726096153259277, "match_error": 0.00010585784912109375, "scalar": 9.721792308220756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1420, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.500004830277803, -0.026251847368671755], "logp": -23.526256677646476, "cache_id": 1421, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.721794128417969, "actual_norm": 9.726202011108398, "relative_norm": 0.04999164931892245, "target_actual_norm": 9.726096153259277, "match_error": 0.00010585784912109375, "scalar": 9.721792308220756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-23.625004830277803, -7.756443837478591e-05], "logp": -23.625082394716177, "cache_id": 1422, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.721794128417969, "actual_norm": 9.726202011108398, "relative_norm": 0.04999164931892245, "target_actual_norm": 9.726096153259277, "match_error": 0.00010585784912109375, "scalar": 9.721792308220756, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.09882571706970111, "first_log_odds": -0.125, "later_log_odds": 0.026174282930298887, "first_differing_log_odds": -0.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-opposite.npz", "array_sha256": "f1bed8949c76dbf46cef02279b8a319fe751a74cb3ab9953ccc40591d3578f3a", "seconds": 1.3350420417264104, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-unrelated", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.724298477172852, "actual_norm": 9.726046562194824, "relative_norm": 0.04999085032795295, "target_actual_norm": 9.726096153259277, "match_error": -4.9591064453125e-05, "scalar": 9.724296700807827, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1423, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.62500761497863, -0.02424492484747953], "logp": -23.64925253982611, "cache_id": 1424, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.724298477172852, "actual_norm": 9.726046562194824, "relative_norm": 0.04999085032795295, "target_actual_norm": 9.726096153259277, "match_error": -4.9591064453125e-05, "scalar": 9.724296700807827, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.75000761497863, -5.754036489378828e-05], "logp": -22.750065155343524, "cache_id": 1425, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.724298477172852, "actual_norm": 9.726046562194824, "relative_norm": 0.04999085032795295, "target_actual_norm": 9.726096153259277, "match_error": -4.9591064453125e-05, "scalar": 9.724296700807827, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.8991873844825875, "first_log_odds": 0.875, "later_log_odds": 0.024187384482587504, "first_differing_log_odds": 0.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-unrelated.npz", "array_sha256": "b3a27893aebf41b31db60f8e8802ed2051e8f244b69da9ff906ecf1524067448", "seconds": 1.3279035002924502, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-ordinary", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727433204650879, "actual_norm": 9.726034164428711, "relative_norm": 0.0499907866047468, "target_actual_norm": 9.726096153259277, "match_error": -6.198883056640625e-05, "scalar": 9.727431829305715, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1426, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.87500784899238, -0.027653083019317373], "logp": -23.902660932011695, "cache_id": 1427, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727433204650879, "actual_norm": 9.726034164428711, "relative_norm": 0.0499907866047468, "target_actual_norm": 9.726096153259277, "match_error": -6.198883056640625e-05, "scalar": 9.727431829305715, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.37500784899238, -5.610665749587029e-05], "logp": -22.375063955649875, "cache_id": 1428, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.727433204650879, "actual_norm": 9.726034164428711, "relative_norm": 0.0499907866047468, "target_actual_norm": 9.726096153259277, "match_error": -6.198883056640625e-05, "scalar": 9.727431829305715, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.5275969763618207, "first_log_odds": 1.5, "later_log_odds": 0.02759697636182068, "first_differing_log_odds": 1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-ordinary.npz", "array_sha256": "11b8bd6b5b786441f5e20a1fc746b2b17d971a80fb18a7afcbcb6018af964843", "seconds": 1.3329205829650164, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-random0", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.733296394348145, "actual_norm": 9.726095199584961, "relative_norm": 0.04999110031899249, "target_actual_norm": 9.726096153259277, "match_error": -9.5367431640625e-07, "scalar": 9.73329396306508, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1429, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.375010812925616, -0.025609642354477092], "logp": -23.400620455280094, "cache_id": 1430, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.733296394348145, "actual_norm": 9.726095199584961, "relative_norm": 0.04999110031899249, "target_actual_norm": 9.726096153259277, "match_error": -9.5367431640625e-07, "scalar": 9.73329396306508, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.375010812925616, -5.227333584481032e-05], "logp": -22.375063086261463, "cache_id": 1431, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.733296394348145, "actual_norm": 9.726095199584961, "relative_norm": 0.04999110031899249, "target_actual_norm": 9.726096153259277, "match_error": -9.5367431640625e-07, "scalar": 9.73329396306508, "random_seed": 20273308, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.0255573690186317, "first_log_odds": 1.0, "later_log_odds": 0.025557369018631704, "first_differing_log_odds": 1.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-random0.npz", "array_sha256": "19e5e529bebfdd0888de96d20e4c1f55bb92dd1da2058e50f9d74496d182bed0", "seconds": 1.328565166797489, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-random1", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.732848167419434, "actual_norm": 9.726094245910645, "relative_norm": 0.0499910954172074, "target_actual_norm": 9.726096153259277, "match_error": -1.9073486328125e-06, "scalar": 9.732848737716267, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1432, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-23.62500749682152, -0.029168202144016202], "logp": -23.65417569896554, "cache_id": 1433, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.732848167419434, "actual_norm": 9.726094245910645, "relative_norm": 0.0499910954172074, "target_actual_norm": 9.726096153259277, "match_error": -1.9073486328125e-06, "scalar": 9.732848737716267, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.75000749682152, -6.243096239354612e-05], "logp": -22.750069927783915, "cache_id": 1434, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.732848167419434, "actual_norm": 9.726094245910645, "relative_norm": 0.0499910954172074, "target_actual_norm": 9.726096153259277, "match_error": -1.9073486328125e-06, "scalar": 9.732848737716267, "random_seed": 20273309, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.9041057711816229, "first_log_odds": 0.875, "later_log_odds": 0.02910577118162294, "first_differing_log_odds": 0.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-random1.npz", "array_sha256": "df278bcfc7d7aa7ed57e73bf42cd4fc545eb03515516ac55245351570ed138a9", "seconds": 1.3243874576874077, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-random2", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.725669860839844, "actual_norm": 9.725990295410156, "relative_norm": 0.049990561122632705, "target_actual_norm": 9.726096153259277, "match_error": -0.00010585784912109375, "scalar": 9.725669478966665, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1435, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-24.000007124389874, -0.023685541568272312], "logp": -24.023692665958148, "cache_id": 1436, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.725669860839844, "actual_norm": 9.725990295410156, "relative_norm": 0.049990561122632705, "target_actual_norm": 9.726096153259277, "match_error": -0.00010585784912109375, "scalar": 9.725669478966665, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.750007124389874, -5.7644965931707835e-05], "logp": -22.750064769355806, "cache_id": 1437, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.55653381347656, "strength_one_norm": 36.8559684753418, "target_shrink_scale": 0.26394169229828157, "requested_norm": 9.725669860839844, "actual_norm": 9.725990295410156, "relative_norm": 0.049990561122632705, "target_actual_norm": 9.726096153259277, "match_error": -0.00010585784912109375, "scalar": 9.725669478966665, "random_seed": 20273310, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.273627896602342, "first_log_odds": 1.25, "later_log_odds": 0.02362789660234199, "first_differing_log_odds": 1.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-random2.npz", "array_sha256": "f95d9292546a19f3ccc09e6030e627f4c57fa66c0fa7264e808061f37768ce3a", "seconds": 1.2963391249068081, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-24-paraphrase-bridge", "case_id": "eval-24", "city": "Eilat", "country": "Israel", "code": "ILS", "target_country": "Sweden", "target_code": "SEK", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.<|im_end|>", "output_text": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "output_ids": [785, 11413, 1483, 304, 468, 321, 266, 374, 279, 15907, 1532, 2932, 17659, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 1715, 50, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 164.03346252441406, 127.2728042602539, 554.89892578125, 110.77892303466797, 32.649757385253906, 222.56326293945312, 91.86856079101562, 75.13612365722656, 66.31683349609375, 3.3127808570861816, 157.83253479003906, 100.88802337646484, 90.99966430664062, 63.83460998535156, 176.7198486328125, 116.35342407226562, 105.98113250732422, 26.916128158569336, 286.89031982421875, 115.13066864013672, 218.28024291992188, 351.8733825683594, 280.6033935546875, 179.77662658691406, 284.463134765625]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 164.06690979003906, 127.19817352294922, 554.8496704101562, 110.81114959716797, 32.64612579345703, 222.57821655273438, 91.88037109375, 75.15412902832031, 66.33618927001953, 3.4141175746917725, 157.84402465820312, 100.85762786865234, 91.04283142089844, 63.84634017944336, 176.70188903808594, 116.36112213134766, 105.97176361083984, 26.9954776763916, 286.888916015625, 115.09405517578125, 218.35765075683594, 351.96661376953125, 280.65362548828125, 179.76702880859375, 284.45452880859375]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.14628951251506805, 0.09682071208953857, 0.556630551815033, 0.11581314355134964, 0.030065396800637245, 0.20628713071346283, 0.08545218408107758, 0.07438891381025314, 0.056425753980875015, 0.0032288306392729282, 0.14919866621494293, 0.10313724726438522, 0.0942147746682167, 0.06284375488758087, 0.16717034578323364, 0.1248852089047432, 0.11982555687427521, 0.031223369762301445, 0.28250205516815186, 0.12215175479650497, 0.19092856347560883, 0.3353997766971588, 0.29539984464645386, 0.1732555478811264, 0.2902712821960449]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1438, "cached": true, "scoring": {"normalized": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **ILS**.", "bare_format": false, "malformed": true, "first_code": "ILS", "all_codes": ["ILS"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Eilat is the Israeli New Shekel, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Israeli", "start": 34, "end": 41}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [1715, 50], "tokens": ["IL", "S"], "log_probs": [-27.687562898318685, -0.055103054438348766], "logp": -27.742665952757033, "cache_id": 1439, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 164.03346252441406, 127.2728042602539, 554.89892578125, 110.77892303466797, 32.649757385253906, 222.56326293945312, 91.86856079101562, 75.13612365722656, 66.31683349609375, 3.3127808570861816, 157.83253479003906, 100.88802337646484, 90.99966430664062, 63.83460998535156, 176.7198486328125, 116.35342407226562, 105.98113250732422, 26.916128158569336, 286.89031982421875, 115.13066864013672, 218.28024291992188, 351.8733825683594, 280.6033935546875, 179.77662658691406, 284.463134765625]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 164.06690979003906, 127.19817352294922, 554.8496704101562, 110.81114959716797, 32.64612579345703, 222.57821655273438, 91.88037109375, 75.15412902832031, 66.33618927001953, 3.4141175746917725, 157.84402465820312, 100.85762786865234, 91.04283142089844, 63.84634017944336, 176.70188903808594, 116.36112213134766, 105.97176361083984, 26.9954776763916, 286.888916015625, 115.09405517578125, 218.35765075683594, 351.96661376953125, 280.65362548828125, 179.76702880859375, 284.45452880859375]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.14628951251506805, 0.09682071208953857, 0.556630551815033, 0.11581314355134964, 0.030065396800637245, 0.20628713071346283, 0.08545218408107758, 0.07438891381025314, 0.056425753980875015, 0.0032288306392729282, 0.14919866621494293, 0.10313724726438522, 0.0942147746682167, 0.06284375488758087, 0.16717034578323364, 0.1248852089047432, 0.11982555687427521, 0.031223369762301445, 0.28250205516815186, 0.12215175479650497, 0.19092856347560883, 0.3353997766971588, 0.29539984464645386, 0.1732555478811264, 0.2902712821960449]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-11.750062898318685, -3.67528150958338e-05], "logp": -11.75009965113378, "cache_id": 1440, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[94.30854797363281, 181.53575134277344, 83.61700439453125, 323.484375, 29.768115997314453, 85.04322052001953, 132.6435546875, 159.14251708984375, 164.03346252441406, 127.2728042602539, 554.89892578125, 110.77892303466797, 32.649757385253906, 222.56326293945312, 91.86856079101562, 75.13612365722656, 66.31683349609375, 3.3127808570861816, 157.83253479003906, 100.88802337646484, 90.99966430664062, 63.83460998535156, 176.7198486328125, 116.35342407226562, 105.98113250732422, 26.916128158569336, 286.89031982421875, 115.13066864013672, 218.28024291992188, 351.8733825683594, 280.6033935546875, 179.77662658691406, 284.463134765625]], "actual_norm": [[94.29801940917969, 181.55288696289062, 83.5897445678711, 323.52789306640625, 29.76899528503418, 85.11279296875, 132.65280151367188, 159.0937957763672, 164.06690979003906, 127.19817352294922, 554.8496704101562, 110.81114959716797, 32.64612579345703, 222.57821655273438, 91.88037109375, 75.15412902832031, 66.33618927001953, 3.4141175746917725, 157.84402465820312, 100.85762786865234, 91.04283142089844, 63.84634017944336, 176.70188903808594, 116.36112213134766, 105.97176361083984, 26.9954776763916, 286.888916015625, 115.09405517578125, 218.35765075683594, 351.96661376953125, 280.65362548828125, 179.76702880859375, 284.45452880859375]], "relative_norm": [[0.005503691732883453, 0.1925874501466751, 0.09247631579637527, 0.2820824980735779, 0.027097979560494423, 0.07394053786993027, 0.11797989904880524, 0.1413240134716034, 0.14628951251506805, 0.09682071208953857, 0.556630551815033, 0.11581314355134964, 0.030065396800637245, 0.20628713071346283, 0.08545218408107758, 0.07438891381025314, 0.056425753980875015, 0.0032288306392729282, 0.14919866621494293, 0.10313724726438522, 0.0942147746682167, 0.06284375488758087, 0.16717034578323364, 0.1248852089047432, 0.11982555687427521, 0.031223369762301445, 0.28250205516815186, 0.12215175479650497, 0.19092856347560883, 0.3353997766971588, 0.29539984464645386, 0.1732555478811264, 0.2902712821960449]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.992566301623253, "first_log_odds": 15.9375, "later_log_odds": 0.055066301623252656, "first_differing_log_odds": 15.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-24-paraphrase-bridge.npz", "array_sha256": "562a2a2aa8e309730173585313cc0c80988ed66b7a5acb6628b95e6de2a30f18", "seconds": 1.3392764581367373, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-baseline", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1441, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00031204060576700007, -1.8713486563530473e-07], "logp": -0.00031222774063263534, "cache_id": 1442, "trace": []}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.750312040605767, -0.03152919871285707], "logp": -23.781841239318624, "cache_id": 1443, "trace": []}, "log_odds": -23.78152901157799, "first_log_odds": -23.75, "later_log_odds": -0.03152901157799093, "first_differing_log_odds": -23.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-baseline.npz", "array_sha256": "41cd90a0d45df87be9106ba364f0d36d48b0e33055b85c4feacf1159ad86503c", "seconds": 0.3885722910054028, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457567232, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-noop", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.468782424926758, "match_error": -9.468782424926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1444, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00031204060576700007, -1.8713486563530473e-07], "logp": -0.00031222774063263534, "cache_id": 1445, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.468782424926758, "match_error": -9.468782424926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.750312040605767, -0.03152919871285707], "logp": -23.781841239318624, "cache_id": 1446, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.468782424926758, "match_error": -9.468782424926758, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.78152901157799, "first_log_odds": -23.75, "later_log_odds": -0.03152901157799093, "first_differing_log_odds": -23.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-noop.npz", "array_sha256": "f4bbbec932baec3908be1f8128f3698eb1b6ac624ed616e2ea0613c4f9d83117", "seconds": 0.3437182498164475, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457600000, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-j", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469208717346191, "actual_norm": 9.468782424926758, "relative_norm": 0.04998435537499697, "target_actual_norm": 9.468782424926758, "match_error": 0.0, "scalar": 0.42096795139618837, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1447, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0005570454727693006, -1.4943957864774348e-07], "logp": -0.0005571949123479484, "cache_id": 1448, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469208717346191, "actual_norm": 9.468782424926758, "relative_norm": 0.04998435537499697, "target_actual_norm": 9.468782424926758, "match_error": 0.0, "scalar": 0.42096795139618837, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.87555704547277, -0.012800786996315486], "logp": -22.888357832469087, "cache_id": 1449, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469208717346191, "actual_norm": 9.468782424926758, "relative_norm": 0.04998435537499697, "target_actual_norm": 9.468782424926758, "match_error": 0.0, "scalar": 0.42096795139618837, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.88780063755674, "first_log_odds": -22.875, "later_log_odds": -0.012800637556740213, "first_differing_log_odds": -22.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-j.npz", "array_sha256": "92fbd256d4d01622478bc6efff9c262ea9e57b3caebe8e67067851be4b4ac745", "seconds": 0.3361172080039978, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457600000, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-opposite", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469218254089355, "actual_norm": 9.47134017944336, "relative_norm": 0.049997857397219306, "target_actual_norm": 9.468782424926758, "match_error": 0.0025577545166015625, "scalar": 9.46921587138786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1450, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00036352678479868543, -1.748999745293383e-07], "logp": -0.00036370168477321477, "cache_id": 1451, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469218254089355, "actual_norm": 9.47134017944336, "relative_norm": 0.049997857397219306, "target_actual_norm": 9.468782424926758, "match_error": 0.0025577545166015625, "scalar": 9.46921587138786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.7503635267848, -0.03149570764544185], "logp": -23.78185923443024, "cache_id": 1452, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.469218254089355, "actual_norm": 9.47134017944336, "relative_norm": 0.049997857397219306, "target_actual_norm": 9.468782424926758, "match_error": 0.0025577545166015625, "scalar": 9.46921587138786, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.781495532745467, "first_log_odds": -23.75, "later_log_odds": -0.0314955327454669, "first_differing_log_odds": -23.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-opposite.npz", "array_sha256": "25c720309108d05c32cd4012650e367b14943a253e17c652c5a2376251efdd48", "seconds": 0.33976608281955123, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457600000, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-unrelated", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.476042747497559, "actual_norm": 9.468927383422852, "relative_norm": 0.04998512059027579, "target_actual_norm": 9.468782424926758, "match_error": 0.00014495849609375, "scalar": 9.476042653150216, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1453, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00038134549566745316, -1.9726759056755698e-07], "logp": -0.0003815427632580207, "cache_id": 1454, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.476042747497559, "actual_norm": 9.468927383422852, "relative_norm": 0.04998512059027579, "target_actual_norm": 9.468782424926758, "match_error": 0.00014495849609375, "scalar": 9.476042653150216, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.56288134549567, -0.017896195927579243], "logp": -23.58077754142325, "cache_id": 1455, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.476042747497559, "actual_norm": 9.468927383422852, "relative_norm": 0.04998512059027579, "target_actual_norm": 9.468782424926758, "match_error": 0.00014495849609375, "scalar": 9.476042653150216, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.58039599865999, "first_log_odds": -23.5625, "later_log_odds": -0.017895998659991363, "first_differing_log_odds": -23.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-unrelated.npz", "array_sha256": "9b0d7f7689a7f5f0ff67fbb45c00415d3496b71ecdeef3d3b82aeb14ac1673e9", "seconds": 0.3381581660360098, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457600000, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-ordinary", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.467817306518555, "actual_norm": 9.469003677368164, "relative_norm": 0.049985523335159376, "target_actual_norm": 9.468782424926758, "match_error": 0.00022125244140625, "scalar": 9.467816200523885, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1456, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0003624286902258705, -1.8087792049220877e-07], "logp": -0.00036260956814636276, "cache_id": 1457, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.467817306518555, "actual_norm": 9.469003677368164, "relative_norm": 0.049985523335159376, "target_actual_norm": 9.468782424926758, "match_error": 0.00022125244140625, "scalar": 9.467816200523885, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.562862428690227, -0.02489643287526968], "logp": -23.587758861565497, "cache_id": 1458, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.467817306518555, "actual_norm": 9.469003677368164, "relative_norm": 0.049985523335159376, "target_actual_norm": 9.468782424926758, "match_error": 0.00022125244140625, "scalar": 9.467816200523885, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.58739625199735, "first_log_odds": -23.5625, "later_log_odds": -0.024896251997351015, "first_differing_log_odds": -23.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-ordinary.npz", "array_sha256": "7da213dc9c5e483636c243e6cfd0d931c1ca0c8004dbc497fbfac52274a81723", "seconds": 0.34338341699913144, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457632768, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-random0", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.463565826416016, "actual_norm": 9.468908309936523, "relative_norm": 0.04998501990405489, "target_actual_norm": 9.468782424926758, "match_error": 0.000125885009765625, "scalar": 9.463563007124321, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1459, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0003488757762004735, -1.6734381967713444e-07], "logp": -0.0003490431200201506, "cache_id": 1460, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.463565826416016, "actual_norm": 9.468908309936523, "relative_norm": 0.04998501990405489, "target_actual_norm": 9.468782424926758, "match_error": 0.000125885009765625, "scalar": 9.463563007124321, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.6878488757762, -0.022111121452779214], "logp": -23.70995999722898, "cache_id": 1461, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.463565826416016, "actual_norm": 9.468908309936523, "relative_norm": 0.04998501990405489, "target_actual_norm": 9.468782424926758, "match_error": 0.000125885009765625, "scalar": 9.463563007124321, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.70961095410896, "first_log_odds": -23.6875, "later_log_odds": -0.022110954108960357, "first_differing_log_odds": -23.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-random0.npz", "array_sha256": "181dad10a0d8eea56389ec4ed0f2e0483745726139bb9f799cf25277c7076228", "seconds": 0.3433522921986878, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457632768, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-random1", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.47398567199707, "actual_norm": 9.468692779541016, "relative_norm": 0.049983882149758754, "target_actual_norm": 9.468782424926758, "match_error": -8.96453857421875e-05, "scalar": 9.473983782459982, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1462, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0003681208219043636, -1.973406942984128e-07], "logp": -0.00036831816259866204, "cache_id": 1463, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.47398567199707, "actual_norm": 9.468692779541016, "relative_norm": 0.049983882149758754, "target_actual_norm": 9.468782424926758, "match_error": -8.96453857421875e-05, "scalar": 9.473983782459982, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.687868120821904, -0.022500171769090745], "logp": -23.710368292590996, "cache_id": 1464, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.47398567199707, "actual_norm": 9.468692779541016, "relative_norm": 0.049983882149758754, "target_actual_norm": 9.468782424926758, "match_error": -8.96453857421875e-05, "scalar": 9.473983782459982, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.709999974428396, "first_log_odds": -23.6875, "later_log_odds": -0.022499974428395575, "first_differing_log_odds": -23.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-random1.npz", "array_sha256": "a3c927535d0e1e9c937d88d9494ab837262942b39b080e9080846fcbaf625446", "seconds": 0.3439809577539563, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457632768, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-random2", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.464194297790527, "actual_norm": 9.468771934509277, "relative_norm": 0.049984299997575476, "target_actual_norm": 9.468782424926758, "match_error": -1.049041748046875e-05, "scalar": 9.464195116546762, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1465, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00036523373281295183, -1.7929341203112327e-07], "logp": -0.00036541302622498294, "cache_id": 1466, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.464194297790527, "actual_norm": 9.468771934509277, "relative_norm": 0.049984299997575476, "target_actual_norm": 9.468782424926758, "match_error": -1.049041748046875e-05, "scalar": 9.464195116546762, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.68786523373281, -0.031039698329912803], "logp": -23.718904932062724, "cache_id": 1467, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 189.43492126464844, "strength_one_norm": 22.4938907623291, "target_shrink_scale": 0.42096795139618837, "requested_norm": 9.464194297790527, "actual_norm": 9.468771934509277, "relative_norm": 0.049984299997575476, "target_actual_norm": 9.468782424926758, "match_error": -1.049041748046875e-05, "scalar": 9.464195116546762, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -23.7185395190365, "first_log_odds": -23.6875, "later_log_odds": -0.031039519036500707, "first_differing_log_odds": -23.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-random2.npz", "array_sha256": "a5d4102291ab8f86fbbe02fe89f880786f23900d51c58d3113748b51dab1bef4", "seconds": 0.3430421673692763, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457632768, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-main-bridge", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": " Switzerland Franc (CHF)<|im_end|>", "output_text": " Switzerland Franc (CHF)", "output_ids": [29121, 9694, 320, 2149, 37, 8, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 668.3399047851562, 284.2750549316406, 238.5177764892578, 100.34140014648438, 355.1248779296875, 498.6395263671875, 476.7779846191406, 546.8684692382812, 170.81871032714844, 250.87550354003906, 332.87530517578125, 500.8936767578125, 121.37620544433594, 172.33175659179688, 233.36502075195312, 225.54678344726562, 136.93594360351562, 113.1686782836914, 86.96416473388672, 9.94847583770752, 32.507137298583984, 121.19871520996094, 117.2574691772461, 4.3761773109436035, 323.74346923828125, 1176.1331787109375]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 668.33837890625, 284.3083190917969, 238.5509490966797, 100.34428405761719, 355.0307922363281, 498.5931396484375, 476.8045959472656, 546.9124755859375, 170.87213134765625, 250.83456420898438, 332.885986328125, 500.8465881347656, 121.34691619873047, 172.3004913330078, 233.37860107421875, 225.54632568359375, 136.9164581298828, 113.14402770996094, 86.97856140136719, 10.004059791564941, 32.54463577270508, 121.2027359008789, 117.28907775878906, 4.582923412322998, 323.7343444824219, 1176.0352783203125]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.6225071549415588, 0.2968067526817322, 0.232662171125412, 0.0957401767373085, 0.34755024313926697, 0.48525455594062805, 0.41719019412994385, 0.5370253324508667, 0.1629939079284668, 0.25497937202453613, 0.3431231379508972, 0.5015552639961243, 0.1173938438296318, 0.1653369665145874, 0.1879560351371765, 0.22793327271938324, 0.15369917452335358, 0.12515082955360413, 0.10358582437038422, 0.009900776669383049, 0.034437328577041626, 0.10703268647193909, 0.11277467757463455, 0.004931550472974777, 0.31444647908210754, 0.9861684441566467]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1468, "cached": true, "scoring": {"normalized": "Switzerland Franc (CHF)", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": " Switzerland Franc (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Switzerland", "start": 1, "end": 12}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-22.828699762944808, -3.285851339634377e-07], "logp": -22.828700091529942, "cache_id": 1469, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 668.3399047851562, 284.2750549316406, 238.5177764892578, 100.34140014648438, 355.1248779296875, 498.6395263671875, 476.7779846191406, 546.8684692382812, 170.81871032714844, 250.87550354003906, 332.87530517578125, 500.8936767578125, 121.37620544433594, 172.33175659179688, 233.36502075195312, 225.54678344726562, 136.93594360351562, 113.1686782836914, 86.96416473388672, 9.94847583770752, 32.507137298583984, 121.19871520996094, 117.2574691772461, 4.3761773109436035, 323.74346923828125, 1176.1331787109375]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 668.33837890625, 284.3083190917969, 238.5509490966797, 100.34428405761719, 355.0307922363281, 498.5931396484375, 476.8045959472656, 546.9124755859375, 170.87213134765625, 250.83456420898438, 332.885986328125, 500.8465881347656, 121.34691619873047, 172.3004913330078, 233.37860107421875, 225.54632568359375, 136.9164581298828, 113.14402770996094, 86.97856140136719, 10.004059791564941, 32.54463577270508, 121.2027359008789, 117.28907775878906, 4.582923412322998, 323.7343444824219, 1176.0352783203125]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.6225071549415588, 0.2968067526817322, 0.232662171125412, 0.0957401767373085, 0.34755024313926697, 0.48525455594062805, 0.41719019412994385, 0.5370253324508667, 0.1629939079284668, 0.25497937202453613, 0.3431231379508972, 0.5015552639961243, 0.1173938438296318, 0.1653369665145874, 0.1879560351371765, 0.22793327271938324, 0.15369917452335358, 0.12515082955360413, 0.10358582437038422, 0.009900776669383049, 0.034437328577041626, 0.10703268647193909, 0.11277467757463455, 0.004931550472974777, 0.31444647908210754, 0.9861684441566467]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-0.984949762944807, -0.009900489086687654], "logp": -0.9948502520314946, "cache_id": 1470, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 668.3399047851562, 284.2750549316406, 238.5177764892578, 100.34140014648438, 355.1248779296875, 498.6395263671875, 476.7779846191406, 546.8684692382812, 170.81871032714844, 250.87550354003906, 332.87530517578125, 500.8936767578125, 121.37620544433594, 172.33175659179688, 233.36502075195312, 225.54678344726562, 136.93594360351562, 113.1686782836914, 86.96416473388672, 9.94847583770752, 32.507137298583984, 121.19871520996094, 117.2574691772461, 4.3761773109436035, 323.74346923828125, 1176.1331787109375]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 668.33837890625, 284.3083190917969, 238.5509490966797, 100.34428405761719, 355.0307922363281, 498.5931396484375, 476.8045959472656, 546.9124755859375, 170.87213134765625, 250.83456420898438, 332.885986328125, 500.8465881347656, 121.34691619873047, 172.3004913330078, 233.37860107421875, 225.54632568359375, 136.9164581298828, 113.14402770996094, 86.97856140136719, 10.004059791564941, 32.54463577270508, 121.2027359008789, 117.28907775878906, 4.582923412322998, 323.7343444824219, 1176.0352783203125]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.6225071549415588, 0.2968067526817322, 0.232662171125412, 0.0957401767373085, 0.34755024313926697, 0.48525455594062805, 0.41719019412994385, 0.5370253324508667, 0.1629939079284668, 0.25497937202453613, 0.3431231379508972, 0.5015552639961243, 0.1173938438296318, 0.1653369665145874, 0.1879560351371765, 0.22793327271938324, 0.15369917452335358, 0.12515082955360413, 0.10358582437038422, 0.009900776669383049, 0.034437328577041626, 0.10703268647193909, 0.11277467757463455, 0.004931550472974777, 0.31444647908210754, 0.9861684441566467]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 21.83384983949845, "first_log_odds": 21.84375, "later_log_odds": -0.009900160501551625, "first_differing_log_odds": 21.84375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-main-bridge.npz", "array_sha256": "5d2e17e89db90e39a89cefef2a5555af37a447885819b447bc537ece17e528a5", "seconds": 0.4700037078000605, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-baseline", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1471, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-5.504546328998332, -5.343565732180623e-07], "logp": -5.504546863354905, "cache_id": 1472, "trace": []}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.25454632899833, -0.00344949995171582], "logp": -23.257995828950047, "cache_id": 1473, "trace": []}, "log_odds": -17.75344896559514, "first_log_odds": -17.75, "later_log_odds": -0.0034489655951404075, "first_differing_log_odds": -17.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-baseline.npz", "array_sha256": "534b6b2f66e3486bb9f97fb7bf366f69c068893776e9fd74c67b5e7f56d44c75", "seconds": 1.2197262500412762, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-noop", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.407812118530273, "match_error": -9.407812118530273, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1474, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-5.504546328998332, -5.343565732180623e-07], "logp": -5.504546863354905, "cache_id": 1475, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.407812118530273, "match_error": -9.407812118530273, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.25454632899833, -0.00344949995171582], "logp": -23.257995828950047, "cache_id": 1476, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.407812118530273, "match_error": -9.407812118530273, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.75344896559514, "first_log_odds": -17.75, "later_log_odds": -0.0034489655951404075, "first_differing_log_odds": -17.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-noop.npz", "array_sha256": "595237e74a1fc4f2df0a2a57bccc2c4a963e54a24515befb916cf2fc66bdab34", "seconds": 1.191936125047505, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-j", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.411686897277832, "actual_norm": 9.407812118530273, "relative_norm": 0.0499794079776344, "target_actual_norm": 9.407812118530273, "match_error": 0.0, "scalar": 0.4861749133406099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1477, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-4.511709016896343, -5.764744287873905e-07], "logp": -4.511709593370772, "cache_id": 1478, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.411686897277832, "actual_norm": 9.407812118530273, "relative_norm": 0.0499794079776344, "target_actual_norm": 9.407812118530273, "match_error": 0.0, "scalar": 0.4861749133406099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.199209016896344, -0.0023590174843446912], "logp": -22.20156803438069, "cache_id": 1479, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.411686897277832, "actual_norm": 9.407812118530273, "relative_norm": 0.0499794079776344, "target_actual_norm": 9.407812118530273, "match_error": 0.0, "scalar": 0.4861749133406099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.689858441009918, "first_log_odds": -17.6875, "later_log_odds": -0.0023584410099175557, "first_differing_log_odds": -17.6875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-j.npz", "array_sha256": "6e37442cd2754225c7b5dae36e0bb27ae986db8920d42fc0c05e24b5802133b6", "seconds": 1.1841597501188517, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-opposite", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.412117958068848, "actual_norm": 9.40774917602539, "relative_norm": 0.04997907359286023, "target_actual_norm": 9.407812118530273, "match_error": -6.29425048828125e-05, "scalar": 9.412118673149962, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1480, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-6.002930620020831, -4.719309348825701e-07], "logp": -6.002931091951766, "cache_id": 1481, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.412117958068848, "actual_norm": 9.40774917602539, "relative_norm": 0.04997907359286023, "target_actual_norm": 9.407812118530273, "match_error": -6.29425048828125e-05, "scalar": 9.412118673149962, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-24.37793062002083, -0.003651609682875629], "logp": -24.381582229703707, "cache_id": 1482, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.412117958068848, "actual_norm": 9.40774917602539, "relative_norm": 0.04997907359286023, "target_actual_norm": 9.407812118530273, "match_error": -6.29425048828125e-05, "scalar": 9.412118673149962, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.37865113775194, "first_log_odds": -18.375, "later_log_odds": -0.003651137751941036, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-opposite.npz", "array_sha256": "9ff7f159282d6b9f267dbadaa33247a5624726a0643294d8ba19cab63804edc8", "seconds": 1.199852625373751, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-unrelated", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.399771690368652, "actual_norm": 9.407817840576172, "relative_norm": 0.04997943837625023, "target_actual_norm": 9.407812118530273, "match_error": 5.7220458984375e-06, "scalar": 9.399773216573521, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1483, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-4.759226552852204, -6.228166652467334e-07], "logp": -4.759227175668869, "cache_id": 1484, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.399771690368652, "actual_norm": 9.407817840576172, "relative_norm": 0.04997943837625023, "target_actual_norm": 9.407812118530273, "match_error": 5.7220458984375e-06, "scalar": 9.399773216573521, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.321726552852205, -0.0026179105948374104], "logp": -23.32434446344704, "cache_id": 1485, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.399771690368652, "actual_norm": 9.407817840576172, "relative_norm": 0.04997943837625023, "target_actual_norm": 9.407812118530273, "match_error": 5.7220458984375e-06, "scalar": 9.399773216573521, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.565117287778172, "first_log_odds": -18.5625, "later_log_odds": -0.0026172877781718284, "first_differing_log_odds": -18.5625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-unrelated.npz", "array_sha256": "064756e42e9bb4ea94072b5af2bb37bd9141e4cb3091bcda933a433d580f871b", "seconds": 1.2039622087031603, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-ordinary", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.402251243591309, "actual_norm": 9.407825469970703, "relative_norm": 0.04997947890773801, "target_actual_norm": 9.407812118530273, "match_error": 1.33514404296875e-05, "scalar": 9.402249485479842, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1486, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-5.255797763350124, -4.7570909420969816e-07], "logp": -5.255798239059218, "cache_id": 1487, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.402251243591309, "actual_norm": 9.407825469970703, "relative_norm": 0.04997947890773801, "target_actual_norm": 9.407812118530273, "match_error": 1.33514404296875e-05, "scalar": 9.402249485479842, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.630797763350124, -0.0021768652858295276], "logp": -23.632974628635953, "cache_id": 1488, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.402251243591309, "actual_norm": 9.407825469970703, "relative_norm": 0.04997947890773801, "target_actual_norm": 9.407812118530273, "match_error": 1.33514404296875e-05, "scalar": 9.402249485479842, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.377176389576736, "first_log_odds": -18.375, "later_log_odds": -0.0021763895767357155, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-ordinary.npz", "array_sha256": "5e48ef4fecd3d0d44834ab08da8b6a3140cbdf5eee8ddaa022671034f73f740a", "seconds": 1.196861082687974, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-random0", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.393171310424805, "actual_norm": 9.407814979553223, "relative_norm": 0.04997942317694232, "target_actual_norm": 9.407812118530273, "match_error": 2.86102294921875e-06, "scalar": 9.393169832823332, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1489, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-5.255741394555123, -6.227879258093723e-07], "logp": -5.255742017343049, "cache_id": 1490, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.393171310424805, "actual_norm": 9.407814979553223, "relative_norm": 0.04997942317694232, "target_actual_norm": 9.407812118530273, "match_error": 2.86102294921875e-06, "scalar": 9.393169832823332, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.505741394555123, -0.003223673276519492], "logp": -23.508965067831642, "cache_id": 1491, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.393171310424805, "actual_norm": 9.407814979553223, "relative_norm": 0.04997942317694232, "target_actual_norm": 9.407812118530273, "match_error": 2.86102294921875e-06, "scalar": 9.393169832823332, "random_seed": 20273408, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.253223050488593, "first_log_odds": -18.25, "later_log_odds": -0.00322305048859306, "first_differing_log_odds": -18.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-random0.npz", "array_sha256": "cdda3f827be5c03e6a42974f81215b8071567d43313f04402f4ee171874947ae", "seconds": 1.1872434997931123, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-random1", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.418435096740723, "actual_norm": 9.40772533416748, "relative_norm": 0.04997894693196093, "target_actual_norm": 9.407812118530273, "match_error": -8.678436279296875e-05, "scalar": 9.418434953258839, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1492, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-4.7593185729890575, -6.113041571872452e-07], "logp": -4.759319184293215, "cache_id": 1493, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.418435096740723, "actual_norm": 9.40772533416748, "relative_norm": 0.04997894693196093, "target_actual_norm": 9.407812118530273, "match_error": -8.678436279296875e-05, "scalar": 9.418434953258839, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.13431857298906, -0.003277292474814719], "logp": -23.137595865463872, "cache_id": 1494, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.418435096740723, "actual_norm": 9.40772533416748, "relative_norm": 0.04997894693196093, "target_actual_norm": 9.407812118530273, "match_error": -8.678436279296875e-05, "scalar": 9.418434953258839, "random_seed": 20273409, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.378276681170657, "first_log_odds": -18.375, "later_log_odds": -0.0032766811706572696, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-random1.npz", "array_sha256": "a614a338de1e3d8d383aad2817843723500a0bdb469c79f24d5cffec2e234d31", "seconds": 1.1935751670971513, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-random2", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.39033317565918, "actual_norm": 9.407699584960938, "relative_norm": 0.049978810138189673, "target_actual_norm": 9.407812118530273, "match_error": -0.0001125335693359375, "scalar": 9.39033468436537, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1495, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-5.007363560432663, -4.6324351505332385e-07], "logp": -5.007364023676177, "cache_id": 1496, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.39033317565918, "actual_norm": 9.407699584960938, "relative_norm": 0.049978810138189673, "target_actual_norm": 9.407812118530273, "match_error": -0.0001125335693359375, "scalar": 9.39033468436537, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.382363560432662, -0.003175265282538771], "logp": -23.3855388257152, "cache_id": 1497, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 188.2337646484375, "strength_one_norm": 19.358646392822266, "target_shrink_scale": 0.4861749133406099, "requested_norm": 9.39033317565918, "actual_norm": 9.407699584960938, "relative_norm": 0.049978810138189673, "target_actual_norm": 9.407812118530273, "match_error": -0.0001125335693359375, "scalar": 9.39033468436537, "random_seed": 20273410, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -18.378174802039023, "first_log_odds": -18.375, "later_log_odds": -0.0031748020390232057, "first_differing_log_odds": -18.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-random2.npz", "array_sha256": "396c399e8c7ba55cafc00d352295a4fe30c0c722e944a7650e2bfbc456f0c2da", "seconds": 1.2023810418322682, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-25-paraphrase-bridge", "case_id": "eval-25", "city": "Stockholm", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.<|im_end|>", "output_text": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "output_ids": [785, 11413, 1483, 304, 52082, 374, 279, 30109, 87932, 64, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 713.541748046875, 197.49729919433594, 142.2295684814453, 34.6603889465332, 346.21331787109375, 508.4782409667969, 466.955810546875, 538.5856323242188, 133.313720703125, 249.02041625976562, 335.6553955078125, 513.7953491210938, 184.7783966064453, 264.8627624511719, 171.73086547851562, 64.01670837402344, 19.713844299316406, 40.395423889160156, 130.28732299804688, 105.82792663574219, 40.04696273803711, 332.554443359375, 643.8696899414062]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 713.5588989257812, 197.5029296875, 142.25897216796875, 34.717132568359375, 346.25054931640625, 508.5467224121094, 467.0146484375, 538.624267578125, 133.3466033935547, 249.0257110595703, 335.6298522949219, 513.7966918945312, 184.75665283203125, 264.82415771484375, 171.7058563232422, 64.06190490722656, 19.774070739746094, 40.347206115722656, 130.32821655273438, 105.8018798828125, 40.04319381713867, 332.6025085449219, 643.8711547851562]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.6659401655197144, 0.19220148026943207, 0.12891384959220886, 0.03260170295834541, 0.31733784079551697, 0.49038395285606384, 0.4021916091442108, 0.5099851489067078, 0.12564289569854736, 0.2508889138698578, 0.3426305055618286, 0.5041175484657288, 0.17455454170703888, 0.2767956852912903, 0.19382648169994354, 0.07386410236358643, 0.01924857497215271, 0.042830727994441986, 0.11341454833745956, 0.1001598984003067, 0.04221741482615471, 0.3206852376461029, 0.6492728590965271]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1498, "cached": true, "scoring": {"normalized": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Stockholm is the Swedish krona, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swedish", "start": 38, "end": 45}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-23.488726009398647, -5.398751087690167e-07], "logp": -23.488726549273757, "cache_id": 1499, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 713.541748046875, 197.49729919433594, 142.2295684814453, 34.6603889465332, 346.21331787109375, 508.4782409667969, 466.955810546875, 538.5856323242188, 133.313720703125, 249.02041625976562, 335.6553955078125, 513.7953491210938, 184.7783966064453, 264.8627624511719, 171.73086547851562, 64.01670837402344, 19.713844299316406, 40.395423889160156, 130.28732299804688, 105.82792663574219, 40.04696273803711, 332.554443359375, 643.8696899414062]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 713.5588989257812, 197.5029296875, 142.25897216796875, 34.717132568359375, 346.25054931640625, 508.5467224121094, 467.0146484375, 538.624267578125, 133.3466033935547, 249.0257110595703, 335.6298522949219, 513.7966918945312, 184.75665283203125, 264.82415771484375, 171.7058563232422, 64.06190490722656, 19.774070739746094, 40.347206115722656, 130.32821655273438, 105.8018798828125, 40.04319381713867, 332.6025085449219, 643.8711547851562]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.6659401655197144, 0.19220148026943207, 0.12891384959220886, 0.03260170295834541, 0.31733784079551697, 0.49038395285606384, 0.4021916091442108, 0.5099851489067078, 0.12564289569854736, 0.2508889138698578, 0.3426305055618286, 0.5041175484657288, 0.17455454170703888, 0.2767956852912903, 0.19382648169994354, 0.07386410236358643, 0.01924857497215271, 0.042830727994441986, 0.11341454833745956, 0.1001598984003067, 0.04221741482615471, 0.3206852376461029, 0.6492728590965271]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-6.176226009398649, -0.0018116891751105667], "logp": -6.1780376985737595, "cache_id": 1500, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 713.541748046875, 197.49729919433594, 142.2295684814453, 34.6603889465332, 346.21331787109375, 508.4782409667969, 466.955810546875, 538.5856323242188, 133.313720703125, 249.02041625976562, 335.6553955078125, 513.7953491210938, 184.7783966064453, 264.8627624511719, 171.73086547851562, 64.01670837402344, 19.713844299316406, 40.395423889160156, 130.28732299804688, 105.82792663574219, 40.04696273803711, 332.554443359375, 643.8696899414062]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 713.5588989257812, 197.5029296875, 142.25897216796875, 34.717132568359375, 346.25054931640625, 508.5467224121094, 467.0146484375, 538.624267578125, 133.3466033935547, 249.0257110595703, 335.6298522949219, 513.7966918945312, 184.75665283203125, 264.82415771484375, 171.7058563232422, 64.06190490722656, 19.774070739746094, 40.347206115722656, 130.32821655273438, 105.8018798828125, 40.04319381713867, 332.6025085449219, 643.8711547851562]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.6659401655197144, 0.19220148026943207, 0.12891384959220886, 0.03260170295834541, 0.31733784079551697, 0.49038395285606384, 0.4021916091442108, 0.5099851489067078, 0.12564289569854736, 0.2508889138698578, 0.3426305055618286, 0.5041175484657288, 0.17455454170703888, 0.2767956852912903, 0.19382648169994354, 0.07386410236358643, 0.01924857497215271, 0.042830727994441986, 0.11341454833745956, 0.1001598984003067, 0.04221741482615471, 0.3206852376461029, 0.6492728590965271]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 17.310688850699997, "first_log_odds": 17.3125, "later_log_odds": -0.0018111493000034784, "first_differing_log_odds": 17.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-25-paraphrase-bridge.npz", "array_sha256": "067e99bf64a93f09587324fff87149825f61e846d8a2d78c43f4d6e86e435a59", "seconds": 1.1961171249859035, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-baseline", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1501, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00026344189995290473, -4.067762736429463e-07], "logp": -0.0002638486762265477, "cache_id": 1502, "trace": []}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.750263441899953, -0.02457711332532233], "logp": -21.774840555225275, "cache_id": 1503, "trace": []}, "log_odds": -21.77457670654905, "first_log_odds": -21.75, "later_log_odds": -0.02457670654904831, "first_differing_log_odds": -21.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-baseline.npz", "array_sha256": "f705b5ffea2082047bfdc0d4101a29d0c0e23f5dcf2f05aef8489d573daa2d21", "seconds": 0.37908474961295724, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-noop", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.703375816345215, "match_error": -9.703375816345215, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1504, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00026344189995290473, -4.067762736429463e-07], "logp": -0.0002638486762265477, "cache_id": 1505, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.703375816345215, "match_error": -9.703375816345215, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.750263441899953, -0.02457711332532233], "logp": -21.774840555225275, "cache_id": 1506, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.703375816345215, "match_error": -9.703375816345215, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.77457670654905, "first_log_odds": -21.75, "later_log_odds": -0.02457670654904831, "first_differing_log_odds": -21.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-noop.npz", "array_sha256": "5b84ecd065d33d0da181b21383812d37c6896cb5ea856c122b123ab46e42ef3f", "seconds": 0.34570820815861225, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-j", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701257705688477, "actual_norm": 9.703375816345215, "relative_norm": 0.049995140391103685, "target_actual_norm": 9.703375816345215, "match_error": 0.0, "scalar": 0.6053119599018104, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1507, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00025408476861161355, -7.406472417051135e-07], "logp": -0.00025482541585331866, "cache_id": 1508, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701257705688477, "actual_norm": 9.703375816345215, "relative_norm": 0.049995140391103685, "target_actual_norm": 9.703375816345215, "match_error": 0.0, "scalar": 0.6053119599018104, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-20.87525408476861, -0.014711210914638338], "logp": -20.889965295683247, "cache_id": 1509, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701257705688477, "actual_norm": 9.703375816345215, "relative_norm": 0.049995140391103685, "target_actual_norm": 9.703375816345215, "match_error": 0.0, "scalar": 0.6053119599018104, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -20.889710470267392, "first_log_odds": -20.875, "later_log_odds": -0.01471047026739214, "first_differing_log_odds": -20.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-j.npz", "array_sha256": "8b0b294dae7556c01f00c6f7da48be75395cbf78d9a5d140e61cb35575c31db0", "seconds": 0.34513012506067753, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-opposite", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699524879455566, "actual_norm": 9.703368186950684, "relative_norm": 0.04999510108183051, "target_actual_norm": 9.703375816345215, "match_error": -7.62939453125e-06, "scalar": 9.699526210461045, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1510, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00025081648746420394, -3.851468788278933e-07], "logp": -0.0002512016343430318, "cache_id": 1511, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699524879455566, "actual_norm": 9.703368186950684, "relative_norm": 0.04999510108183051, "target_actual_norm": 9.703375816345215, "match_error": -7.62939453125e-06, "scalar": 9.699526210461045, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.125250816487465, -0.01920725920387002], "logp": -22.144458075691336, "cache_id": 1512, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699524879455566, "actual_norm": 9.703368186950684, "relative_norm": 0.04999510108183051, "target_actual_norm": 9.703375816345215, "match_error": -7.62939453125e-06, "scalar": 9.699526210461045, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -22.144206874056994, "first_log_odds": -22.125, "later_log_odds": -0.019206874056994394, "first_differing_log_odds": -22.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-opposite.npz", "array_sha256": "e38043eec31207d7de673a143381499000d12507f5aaba5287d949613d1bf931", "seconds": 0.33904479211196303, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-unrelated", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.71414566040039, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.714147309732653, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1513, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0002345727039602004, -4.890095927137324e-07], "logp": -0.00023506171355291413, "cache_id": 1514, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.71414566040039, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.714147309732653, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.75023457270396, -0.015599727564598119], "logp": -21.76583430026856, "cache_id": 1515, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.71414566040039, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.714147309732653, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.765599238555005, "first_log_odds": -21.75, "later_log_odds": -0.015599238555005002, "first_differing_log_odds": -21.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-unrelated.npz", "array_sha256": "b071878aabeac413806c305f0f37d3d367d44860ac4acfcdb1f9441edde58621", "seconds": 0.34347704192623496, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-ordinary", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701154708862305, "actual_norm": 9.703472137451172, "relative_norm": 0.049995636670677523, "target_actual_norm": 9.703375816345215, "match_error": 9.632110595703125e-05, "scalar": 9.701154889873578, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1516, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00024208811739756157, -5.074145524264366e-07], "logp": -0.000242595531949988, "cache_id": 1517, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701154708862305, "actual_norm": 9.703472137451172, "relative_norm": 0.049995636670677523, "target_actual_norm": 9.703375816345215, "match_error": 9.632110595703125e-05, "scalar": 9.701154889873578, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.750242088117396, -0.015133003168701621], "logp": -21.7653750912861, "cache_id": 1518, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.701154708862305, "actual_norm": 9.703472137451172, "relative_norm": 0.049995636670677523, "target_actual_norm": 9.703375816345215, "match_error": 9.632110595703125e-05, "scalar": 9.701154889873578, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.76513249575415, "first_log_odds": -21.75, "later_log_odds": -0.015132495754148323, "first_differing_log_odds": -21.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-ordinary.npz", "array_sha256": "7aeeb77265156e1a6c38e777750b0a3d4ea91df3ae39110a9751e3e04a36690b", "seconds": 0.340620166156441, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-random0", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.685358047485352, "actual_norm": 9.703429222106934, "relative_norm": 0.04999541555601591, "target_actual_norm": 9.703375816345215, "match_error": 5.340576171875e-05, "scalar": 9.685358550344063, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1519, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.00023485471089194812, -4.754197251520397e-07], "logp": -0.00023533013061710016, "cache_id": 1520, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.685358047485352, "actual_norm": 9.703429222106934, "relative_norm": 0.04999541555601591, "target_actual_norm": 9.703375816345215, "match_error": 5.340576171875e-05, "scalar": 9.685358550344063, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.500234854710893, -0.018184134877353788], "logp": -21.51841898958825, "cache_id": 1521, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.685358047485352, "actual_norm": 9.703429222106934, "relative_norm": 0.04999541555601591, "target_actual_norm": 9.703375816345215, "match_error": 5.340576171875e-05, "scalar": 9.685358550344063, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.51818365945763, "first_log_odds": -21.5, "later_log_odds": -0.01818365945763034, "first_differing_log_odds": -21.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-random0.npz", "array_sha256": "cd68db1ec322b00502f1e73171376194625426fb535af1520be1bdf679fef360", "seconds": 0.34201670810580254, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-random1", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699601173400879, "actual_norm": 9.70344066619873, "relative_norm": 0.04999547451992568, "target_actual_norm": 9.703375816345215, "match_error": 6.4849853515625e-05, "scalar": 9.699600241343433, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1522, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0002664622530363688, -4.954720076888285e-07], "logp": -0.00026695772504405763, "cache_id": 1523, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699601173400879, "actual_norm": 9.70344066619873, "relative_norm": 0.04999547451992568, "target_actual_norm": 9.703375816345215, "match_error": 6.4849853515625e-05, "scalar": 9.699600241343433, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.875266462253038, -0.015910400841122302], "logp": -21.89117686309416, "cache_id": 1524, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.699601173400879, "actual_norm": 9.70344066619873, "relative_norm": 0.04999547451992568, "target_actual_norm": 9.703375816345215, "match_error": 6.4849853515625e-05, "scalar": 9.699600241343433, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.890909905369114, "first_log_odds": -21.875, "later_log_odds": -0.01590990536911363, "first_differing_log_odds": -21.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-random1.npz", "array_sha256": "21b49b23f23ef83181710ebb5c2273c3988b5babdcc4b772012bad74b1b2c821", "seconds": 0.3398112081922591, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-random2", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SEK<|im_end|>", "output_text": "SEK", "output_ids": [925, 42, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.700933456420898, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.700932797226415, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1525, "cached": true, "scoring": {"normalized": "SEK", "bare_format": true, "malformed": false, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-0.0002632052273553656, -4.4082171033947477e-07], "logp": -0.0002636460490657051, "cache_id": 1526, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.700933456420898, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.700932797226415, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.875263205227355, -0.019246459209714538], "logp": -21.89450966443707, "cache_id": 1527, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 194.0863800048828, "strength_one_norm": 16.026874542236328, "target_shrink_scale": 0.6053119599018104, "requested_norm": 9.700933456420898, "actual_norm": 9.703380584716797, "relative_norm": 0.049995164959399416, "target_actual_norm": 9.703375816345215, "match_error": 4.76837158203125e-06, "scalar": 9.700932797226415, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -21.894246018388003, "first_log_odds": -21.875, "later_log_odds": -0.019246018388002994, "first_differing_log_odds": -21.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-random2.npz", "array_sha256": "c66db09e27c2014e29cd0e1169cf8b4e1382856cccf60e32c1c7f7007ffe711f", "seconds": 0.3430700418539345, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-main-bridge", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 146.26156616210938, 580.8147583007812, 751.0180053710938, 285.89447021484375, 239.00831604003906, 81.72367095947266, 339.0686340332031, 530.3307495117188, 482.332275390625, 529.38720703125, 148.0924072265625, 254.46463012695312, 328.3489685058594, 505.4874572753906, 117.23818969726562, 189.12875366210938, 223.13833618164062, 238.72947692871094, 157.26373291015625, 100.21700286865234, 72.19764709472656, 11.075338363647461, 37.13718032836914, 88.99087524414062, 116.07601928710938, 4.540595054626465, 332.7364196777344, 1044.5174560546875]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 146.30743408203125, 580.7518310546875, 751.0382080078125, 285.9364929199219, 239.01609802246094, 81.69027709960938, 339.1225891113281, 530.4407348632812, 482.3732604980469, 529.371826171875, 148.08395385742188, 254.42066955566406, 328.35540771484375, 505.4432067871094, 117.20109558105469, 189.13705444335938, 223.1123046875, 238.74363708496094, 157.24578857421875, 100.2020263671875, 72.20530700683594, 11.163369178771973, 37.17198181152344, 88.92013549804688, 116.08255004882812, 4.754150390625, 332.8004150390625, 1044.4730224609375]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.1320367455482483, 0.48585331439971924, 0.721984326839447, 0.29794347286224365, 0.23488293588161469, 0.07690570503473282, 0.33000537753105164, 0.5217660665512085, 0.4289245009422302, 0.5278165936470032, 0.1427895873785019, 0.25890642404556274, 0.3401968777179718, 0.5193342566490173, 0.11362609267234802, 0.17923785746097565, 0.18010073900222778, 0.24096572399139404, 0.17849332094192505, 0.11157305538654327, 0.08728121221065521, 0.011112990789115429, 0.03954991325736046, 0.0791584700345993, 0.11270882189273834, 0.005175341852009296, 0.32265451550483704, 0.9029879570007324]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1528, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-23.0898488920404, -3.508813941469469e-06], "logp": -23.08985240085434, "cache_id": 1529, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 146.26156616210938, 580.8147583007812, 751.0180053710938, 285.89447021484375, 239.00831604003906, 81.72367095947266, 339.0686340332031, 530.3307495117188, 482.332275390625, 529.38720703125, 148.0924072265625, 254.46463012695312, 328.3489685058594, 505.4874572753906, 117.23818969726562, 189.12875366210938, 223.13833618164062, 238.72947692871094, 157.26373291015625, 100.21700286865234, 72.19764709472656, 11.075338363647461, 37.13718032836914, 88.99087524414062, 116.07601928710938, 4.540595054626465, 332.7364196777344, 1044.5174560546875]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 146.30743408203125, 580.7518310546875, 751.0382080078125, 285.9364929199219, 239.01609802246094, 81.69027709960938, 339.1225891113281, 530.4407348632812, 482.3732604980469, 529.371826171875, 148.08395385742188, 254.42066955566406, 328.35540771484375, 505.4432067871094, 117.20109558105469, 189.13705444335938, 223.1123046875, 238.74363708496094, 157.24578857421875, 100.2020263671875, 72.20530700683594, 11.163369178771973, 37.17198181152344, 88.92013549804688, 116.08255004882812, 4.754150390625, 332.8004150390625, 1044.4730224609375]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.1320367455482483, 0.48585331439971924, 0.721984326839447, 0.29794347286224365, 0.23488293588161469, 0.07690570503473282, 0.33000537753105164, 0.5217660665512085, 0.4289245009422302, 0.5278165936470032, 0.1427895873785019, 0.25890642404556274, 0.3401968777179718, 0.5193342566490173, 0.11362609267234802, 0.17923785746097565, 0.18010073900222778, 0.24096572399139404, 0.17849332094192505, 0.11157305538654327, 0.08728121221065521, 0.011112990789115429, 0.03954991325736046, 0.0791584700345993, 0.11270882189273834, 0.005175341852009296, 0.32265451550483704, 0.9029879570007324]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-0.12109889204039909, -0.01767120581651373], "logp": -0.13877009785691283, "cache_id": 1530, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 149.61819458007812, 435.12396240234375, 355.2126159667969, 238.88616943359375, 319.16583251953125, 146.26156616210938, 580.8147583007812, 751.0180053710938, 285.89447021484375, 239.00831604003906, 81.72367095947266, 339.0686340332031, 530.3307495117188, 482.332275390625, 529.38720703125, 148.0924072265625, 254.46463012695312, 328.3489685058594, 505.4874572753906, 117.23818969726562, 189.12875366210938, 223.13833618164062, 238.72947692871094, 157.26373291015625, 100.21700286865234, 72.19764709472656, 11.075338363647461, 37.13718032836914, 88.99087524414062, 116.07601928710938, 4.540595054626465, 332.7364196777344, 1044.5174560546875]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 149.64468383789062, 435.1155090332031, 355.23712158203125, 238.89053344726562, 319.2118835449219, 146.30743408203125, 580.7518310546875, 751.0382080078125, 285.9364929199219, 239.01609802246094, 81.69027709960938, 339.1225891113281, 530.4407348632812, 482.3732604980469, 529.371826171875, 148.08395385742188, 254.42066955566406, 328.35540771484375, 505.4432067871094, 117.20109558105469, 189.13705444335938, 223.1123046875, 238.74363708496094, 157.24578857421875, 100.2020263671875, 72.20530700683594, 11.163369178771973, 37.17198181152344, 88.92013549804688, 116.08255004882812, 4.754150390625, 332.8004150390625, 1044.4730224609375]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.1373685598373413, 0.38176223635673523, 0.28900423645973206, 0.20447124540805817, 0.2812897562980652, 0.1320367455482483, 0.48585331439971924, 0.721984326839447, 0.29794347286224365, 0.23488293588161469, 0.07690570503473282, 0.33000537753105164, 0.5217660665512085, 0.4289245009422302, 0.5278165936470032, 0.1427895873785019, 0.25890642404556274, 0.3401968777179718, 0.5193342566490173, 0.11362609267234802, 0.17923785746097565, 0.18010073900222778, 0.24096572399139404, 0.17849332094192505, 0.11157305538654327, 0.08728121221065521, 0.011112990789115429, 0.03954991325736046, 0.0791584700345993, 0.11270882189273834, 0.005175341852009296, 0.32265451550483704, 0.9029879570007324]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 22.951082302997428, "first_log_odds": 22.96875, "later_log_odds": -0.017667697002572424, "first_differing_log_odds": 22.96875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-main-bridge.npz", "array_sha256": "ef5f1941cadb9c1821e1dde198333e488c704166a6f9edfc3bf4ffb00eaac191", "seconds": 0.3324783337302506, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-baseline", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1531, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-6.751196068301886, -1.7754586347234507e-06], "logp": -6.7511978437605205, "cache_id": 1532, "trace": []}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.376196068301887, -0.0012456578064146814], "logp": -22.3774417261083, "cache_id": 1533, "trace": []}, "log_odds": -15.626243882347781, "first_log_odds": -15.625, "later_log_odds": -0.0012438823477811667, "first_differing_log_odds": -15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-baseline.npz", "array_sha256": "4232e14a332384a89425522f49c49a8edede59d07f52a87c3c0ee6a52fc36e39", "seconds": 1.121717999689281, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-noop", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.719046592712402, "match_error": -9.719046592712402, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1534, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-6.751196068301886, -1.7754586347234507e-06], "logp": -6.7511978437605205, "cache_id": 1535, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.719046592712402, "match_error": -9.719046592712402, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.376196068301887, -0.0012456578064146814], "logp": -22.3774417261083, "cache_id": 1536, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.719046592712402, "match_error": -9.719046592712402, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.626243882347781, "first_log_odds": -15.625, "later_log_odds": -0.0012438823477811667, "first_differing_log_odds": -15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-noop.npz", "array_sha256": "d6eaa0c67ce50cba5910b0558f25600d40c464985d88276b00afd65aad652503", "seconds": 1.0913400826975703, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-j", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71519947052002, "actual_norm": 9.719046592712402, "relative_norm": 0.049997141803516686, "target_actual_norm": 9.719046592712402, "match_error": 0.0, "scalar": 0.6948302685466906, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1537, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.125315871295975, -2.037668343238382e-06], "logp": -8.125317908964318, "cache_id": 1538, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71519947052002, "actual_norm": 9.719046592712402, "relative_norm": 0.049997141803516686, "target_actual_norm": 9.719046592712402, "match_error": 0.0, "scalar": 0.6948302685466906, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-21.625315871295975, -0.0009555200527819476], "logp": -21.626271391348755, "cache_id": 1539, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71519947052002, "actual_norm": 9.719046592712402, "relative_norm": 0.049997141803516686, "target_actual_norm": 9.719046592712402, "match_error": 0.0, "scalar": 0.6948302685466906, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -13.500953482384437, "first_log_odds": -13.5, "later_log_odds": -0.0009534823844372653, "first_differing_log_odds": -13.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-j.npz", "array_sha256": "85e43ef956743e5aefe041db70f126c55598ccb8bfffb6cf0af801ee81d4d706", "seconds": 1.091439957730472, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-opposite", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.705255508422852, "actual_norm": 9.719398498535156, "relative_norm": 0.04999895209274144, "target_actual_norm": 9.719046592712402, "match_error": 0.00035190582275390625, "scalar": 9.70525461070065, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1540, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.000348949651455, -1.9873067910763513e-06], "logp": -8.000350936958245, "cache_id": 1541, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.705255508422852, "actual_norm": 9.719398498535156, "relative_norm": 0.04999895209274144, "target_actual_norm": 9.719046592712402, "match_error": 0.00035190582275390625, "scalar": 9.70525461070065, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.750348949651453, -0.001644445926391374], "logp": -23.751993395577845, "cache_id": 1542, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.705255508422852, "actual_norm": 9.719398498535156, "relative_norm": 0.04999895209274144, "target_actual_norm": 9.719046592712402, "match_error": 0.00035190582275390625, "scalar": 9.70525461070065, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.7516424586196, "first_log_odds": -15.749999999999998, "later_log_odds": -0.001642458619601328, "first_differing_log_odds": -15.749999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-opposite.npz", "array_sha256": "f50d9b8d47292b3801f1a4b0008f03f23cab06bbc17d04c77631a6d4ea076c46", "seconds": 1.0950473328121006, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-unrelated", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.716663360595703, "actual_norm": 9.718979835510254, "relative_norm": 0.04999679838821659, "target_actual_norm": 9.719046592712402, "match_error": -6.67572021484375e-05, "scalar": 9.716663351197099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1543, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.000351732191074, -1.7667631666797039e-06], "logp": -8.00035349895424, "cache_id": 1544, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.716663360595703, "actual_norm": 9.718979835510254, "relative_norm": 0.04999679838821659, "target_actual_norm": 9.719046592712402, "match_error": -6.67572021484375e-05, "scalar": 9.716663351197099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.375351732191074, -0.0012230361466571275], "logp": -23.376574768337733, "cache_id": 1545, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.716663360595703, "actual_norm": 9.718979835510254, "relative_norm": 0.04999679838821659, "target_actual_norm": 9.719046592712402, "match_error": -6.67572021484375e-05, "scalar": 9.716663351197099, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.376221269383493, "first_log_odds": -15.375, "later_log_odds": -0.0012212693834925403, "first_differing_log_odds": -15.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-unrelated.npz", "array_sha256": "9717c47f85f482a4373f27a77de481f0c3cb533b14bc25647f8b4d4b25d3346a", "seconds": 1.0942977499216795, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-ordinary", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 11413, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 320, 8206, 374, 23190, 8, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.713336944580078, "actual_norm": 9.719015121459961, "relative_norm": 0.04999697990773235, "target_actual_norm": 9.719046592712402, "match_error": -3.147125244140625e-05, "scalar": 9.7133370087613, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1546, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Sweden", "start": 83, "end": 89}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-7.500575654496868, -1.7478853558994558e-06], "logp": -7.500577402382223, "cache_id": 1547, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.713336944580078, "actual_norm": 9.719015121459961, "relative_norm": 0.04999697990773235, "target_actual_norm": 9.719046592712402, "match_error": -3.147125244140625e-05, "scalar": 9.7133370087613, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.250575654496867, -0.0011997332286104807], "logp": -22.251775387725477, "cache_id": 1548, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.713336944580078, "actual_norm": 9.719015121459961, "relative_norm": 0.04999697990773235, "target_actual_norm": 9.719046592712402, "match_error": -3.147125244140625e-05, "scalar": 9.7133370087613, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.751197985343254, "first_log_odds": -14.75, "later_log_odds": -0.0011979853432535492, "first_differing_log_odds": -14.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-ordinary.npz", "array_sha256": "8c84668edb96a83c8203b37d6a58caa936f11c4bc347deef2f9030900d6681eb", "seconds": 1.307159083429724, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-random0", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.704957008361816, "actual_norm": 9.719098091125488, "relative_norm": 0.04999740672389104, "target_actual_norm": 9.719046592712402, "match_error": 5.14984130859375e-05, "scalar": 9.704958008936956, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1549, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-7.500574055491232, -1.7689866343779249e-06], "logp": -7.500575824477867, "cache_id": 1550, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.704957008361816, "actual_norm": 9.719098091125488, "relative_norm": 0.04999740672389104, "target_actual_norm": 9.719046592712402, "match_error": 5.14984130859375e-05, "scalar": 9.704958008936956, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-22.87557405549123, -0.0011375622550143494], "logp": -22.876711617746246, "cache_id": 1551, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.704957008361816, "actual_norm": 9.719098091125488, "relative_norm": 0.04999740672389104, "target_actual_norm": 9.719046592712402, "match_error": 5.14984130859375e-05, "scalar": 9.704958008936956, "random_seed": 20273508, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.376135793268379, "first_log_odds": -15.375, "later_log_odds": -0.0011357932683786487, "first_differing_log_odds": -15.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-random0.npz", "array_sha256": "b0c2afe0fad810047a1dddf09b152ab85146d83c3fd556cc6e09dee3d3fb4acf", "seconds": 1.1043340000323951, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-random1", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71474552154541, "actual_norm": 9.719233512878418, "relative_norm": 0.04999810336635693, "target_actual_norm": 9.719046592712402, "match_error": 0.000186920166015625, "scalar": 9.714745867138845, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1552, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-7.750448507632431, -1.53743298612455e-06], "logp": -7.750450045065417, "cache_id": 1553, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71474552154541, "actual_norm": 9.719233512878418, "relative_norm": 0.04999810336635693, "target_actual_norm": 9.719046592712402, "match_error": 0.000186920166015625, "scalar": 9.714745867138845, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.00044850763243, -0.0014186906260327595], "logp": -23.001867198258463, "cache_id": 1554, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.71474552154541, "actual_norm": 9.719233512878418, "relative_norm": 0.04999810336635693, "target_actual_norm": 9.719046592712402, "match_error": 0.000186920166015625, "scalar": 9.714745867138845, "random_seed": 20273509, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.251417153193046, "first_log_odds": -15.25, "later_log_odds": -0.001417153193045806, "first_differing_log_odds": -15.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-random1.npz", "array_sha256": "a72f473d49fef23fbab2fdcdbbcb4ce5b1c248e7fbf32baa7d82296c95c46842", "seconds": 1.0890447082929313, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-random2", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 11413, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 320, 8206, 374, 23190, 8, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.719677925109863, "actual_norm": 9.719094276428223, "relative_norm": 0.04999738710015961, "target_actual_norm": 9.719046592712402, "match_error": 4.76837158203125e-05, "scalar": 9.719676871460251, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1555, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 currency code for the currency used in Uppsala (which is Sweden) is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Sweden", "start": 83, "end": 89}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-8.50021660050337, -1.7569413874970128e-06], "logp": -8.500218357444757, "cache_id": 1556, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.719677925109863, "actual_norm": 9.719094276428223, "relative_norm": 0.04999738710015961, "target_actual_norm": 9.719046592712402, "match_error": 4.76837158203125e-05, "scalar": 9.719676871460251, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-23.25021660050337, -0.0013061579119699903], "logp": -23.25152275841534, "cache_id": 1557, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 194.3920440673828, "strength_one_norm": 13.982118606567383, "target_shrink_scale": 0.6948302685466906, "requested_norm": 9.719677925109863, "actual_norm": 9.719094276428223, "relative_norm": 0.04999738710015961, "target_actual_norm": 9.719046592712402, "match_error": 4.76837158203125e-05, "scalar": 9.719676871460251, "random_seed": 20273510, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.751304400970584, "first_log_odds": -14.750000000000002, "later_log_odds": -0.0013044009705822646, "first_differing_log_odds": -14.750000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-random2.npz", "array_sha256": "dbc2d738388244f26ede1861b23c05a58ac29ae04ff01cb0cd562ebe3bd0e870", "seconds": 1.2720580408349633, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-26-paraphrase-bridge", "case_id": "eval-26", "city": "Uppsala", "country": "Sweden", "code": "SEK", "target_country": "Switzerland", "target_code": "CHF", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.<|im_end|>", "output_text": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "output_ids": [785, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 11413, 1483, 304, 547, 59543, 6053, 374, 3070, 925, 42, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 136.85806274414062, 601.9466552734375, 755.4158325195312, 165.49703979492188, 119.54444122314453, 41.33651351928711, 304.5475158691406, 475.6620178222656, 455.0539855957031, 506.5516662597656, 129.859130859375, 244.84376525878906, 342.1559753417969, 520.944580078125, 182.3095703125, 245.8849334716797, 166.47877502441406, 67.59524536132812, 20.29725456237793, 34.118709564208984, 140.4322967529297, 127.21845245361328, 21.851621627807617, 335.5052490234375, 449.655517578125]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 136.9221649169922, 601.951904296875, 755.4183349609375, 165.4449920654297, 119.52149200439453, 41.41403579711914, 304.4905090332031, 475.67889404296875, 455.0354309082031, 506.5938415527344, 129.90794372558594, 244.88636779785156, 342.2008361816406, 520.9072265625, 182.30416870117188, 245.86984252929688, 166.44384765625, 67.55327606201172, 20.337711334228516, 34.11921691894531, 140.50543212890625, 127.15230560302734, 21.874454498291016, 335.5075378417969, 449.63287353515625]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.12610751390457153, 0.5020045042037964, 0.7302551865577698, 0.17016804218292236, 0.11091500520706177, 0.03977522626519203, 0.27703484892845154, 0.47165754437446594, 0.3988378942012787, 0.4869382381439209, 0.12251275777816772, 0.24827247858047485, 0.35184064507484436, 0.5154257416725159, 0.1742788702249527, 0.26145705580711365, 0.18832823634147644, 0.07730597257614136, 0.0199657641351223, 0.0363832525908947, 0.12321405857801437, 0.12080161273479462, 0.023052165284752846, 0.3242683410644531, 0.45364466309547424]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1558, "cached": true, "scoring": {"normalized": "The three-letter ISO 4217 code for the currency used in Uppsala is **SEK**.", "bare_format": false, "malformed": true, "first_code": "SEK", "all_codes": ["SEK"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The three-letter ISO 4217 code for the currency used in Uppsala is **", "substantive_prefix_before_code": true, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [925, 42], "tokens": ["SE", "K"], "log_probs": [-21.25658941554117, -1.6440703615073285e-06], "logp": -21.256591059611534, "cache_id": 1559, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 136.85806274414062, 601.9466552734375, 755.4158325195312, 165.49703979492188, 119.54444122314453, 41.33651351928711, 304.5475158691406, 475.6620178222656, 455.0539855957031, 506.5516662597656, 129.859130859375, 244.84376525878906, 342.1559753417969, 520.944580078125, 182.3095703125, 245.8849334716797, 166.47877502441406, 67.59524536132812, 20.29725456237793, 34.118709564208984, 140.4322967529297, 127.21845245361328, 21.851621627807617, 335.5052490234375, 449.655517578125]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 136.9221649169922, 601.951904296875, 755.4183349609375, 165.4449920654297, 119.52149200439453, 41.41403579711914, 304.4905090332031, 475.67889404296875, 455.0354309082031, 506.5938415527344, 129.90794372558594, 244.88636779785156, 342.2008361816406, 520.9072265625, 182.30416870117188, 245.86984252929688, 166.44384765625, 67.55327606201172, 20.337711334228516, 34.11921691894531, 140.50543212890625, 127.15230560302734, 21.874454498291016, 335.5075378417969, 449.63287353515625]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.12610751390457153, 0.5020045042037964, 0.7302551865577698, 0.17016804218292236, 0.11091500520706177, 0.03977522626519203, 0.27703484892845154, 0.47165754437446594, 0.3988378942012787, 0.4869382381439209, 0.12251275777816772, 0.24827247858047485, 0.35184064507484436, 0.5154257416725159, 0.1742788702249527, 0.26145705580711365, 0.18832823634147644, 0.07730597257614136, 0.0199657641351223, 0.0363832525908947, 0.12321405857801437, 0.12080161273479462, 0.023052165284752846, 0.3242683410644531, 0.45364466309547424]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-5.506589415541173, -0.0011907695769397487], "logp": -5.507780185118112, "cache_id": 1560, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[72.56814575195312, 68.15165710449219, 118.16055297851562, 22.89203643798828, 17.567611694335938, 401.2178039550781, 245.24777221679688, 278.727294921875, 136.85806274414062, 601.9466552734375, 755.4158325195312, 165.49703979492188, 119.54444122314453, 41.33651351928711, 304.5475158691406, 475.6620178222656, 455.0539855957031, 506.5516662597656, 129.859130859375, 244.84376525878906, 342.1559753417969, 520.944580078125, 182.3095703125, 245.8849334716797, 166.47877502441406, 67.59524536132812, 20.29725456237793, 34.118709564208984, 140.4322967529297, 127.21845245361328, 21.851621627807617, 335.5052490234375, 449.655517578125]], "actual_norm": [[72.5735855102539, 68.16787719726562, 118.15113830566406, 22.98248863220215, 17.650278091430664, 401.2411193847656, 245.3038330078125, 278.75860595703125, 136.9221649169922, 601.951904296875, 755.4183349609375, 165.4449920654297, 119.52149200439453, 41.41403579711914, 304.4905090332031, 475.67889404296875, 455.0354309082031, 506.5938415527344, 129.90794372558594, 244.88636779785156, 342.2008361816406, 520.9072265625, 182.30416870117188, 245.86984252929688, 166.44384765625, 67.55327606201172, 20.337711334228516, 34.11921691894531, 140.50543212890625, 127.15230560302734, 21.874454498291016, 335.5075378417969, 449.63287353515625]], "relative_norm": [[0.004235748201608658, 0.07231103628873825, 0.13071198761463165, 0.02003832720220089, 0.01606661081314087, 0.34857258200645447, 0.21817044913768768, 0.247623011469841, 0.12610751390457153, 0.5020045042037964, 0.7302551865577698, 0.17016804218292236, 0.11091500520706177, 0.03977522626519203, 0.27703484892845154, 0.47165754437446594, 0.3988378942012787, 0.4869382381439209, 0.12251275777816772, 0.24827247858047485, 0.35184064507484436, 0.5154257416725159, 0.1742788702249527, 0.26145705580711365, 0.18832823634147644, 0.07730597257614136, 0.0199657641351223, 0.0363832525908947, 0.12321405857801437, 0.12080161273479462, 0.023052165284752846, 0.3242683410644531, 0.45364466309547424]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 15.748810874493422, "first_log_odds": 15.749999999999998, "later_log_odds": -0.0011891255065759054, "first_differing_log_odds": 15.749999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-26-paraphrase-bridge.npz", "array_sha256": "978efd45c7c53cd1d66634da8beeef30e57c76e11bccb8a57ac20432415a1aba", "seconds": 1.1085133752785623, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-baseline", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1561, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.393197183213548e-05, -0.0002627756611042958], "logp": -0.0003367076329364313, "cache_id": 1562, "trace": []}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.125073931971833, -0.07866819137807722], "logp": -29.20374212334991, "cache_id": 1563, "trace": []}, "log_odds": -29.203405415716976, "first_log_odds": -29.125, "later_log_odds": -0.07840541571697557, "first_differing_log_odds": -29.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-baseline.npz", "array_sha256": "71724fb8766ccac370e8729d0441715623fb8f354a95806db65959015b6a2bb3", "seconds": 0.37938387505710125, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457665536, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-noop", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.043149948120117, "match_error": -10.043149948120117, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1564, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.393197183213548e-05, -0.0002627756611042958], "logp": -0.0003367076329364313, "cache_id": 1565, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.043149948120117, "match_error": -10.043149948120117, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.125073931971833, -0.07866819137807722], "logp": -29.20374212334991, "cache_id": 1566, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.043149948120117, "match_error": -10.043149948120117, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.203405415716976, "first_log_odds": -29.125, "later_log_odds": -0.07840541571697557, "first_differing_log_odds": -29.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-noop.npz", "array_sha256": "05513b982cc17f7aac8ef06f38700e4c2ed6a530612ad432cfe330c9bf386a69", "seconds": 0.33618299989029765, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457698304, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-j", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.040351867675781, "actual_norm": 10.043149948120117, "relative_norm": 0.04999438103544965, "target_actual_norm": 10.043149948120117, "match_error": 0.0, "scalar": 0.17713174964968376, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1567, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-0.00019151798981940587, -0.0008814279409084811], "logp": -0.001072945930727887, "cache_id": 1568, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.040351867675781, "actual_norm": 10.043149948120117, "relative_norm": 0.04999438103544965, "target_actual_norm": 10.043149948120117, "match_error": 0.0, "scalar": 0.17713174964968376, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-27.75019151798982, -0.08008552876499143], "logp": -27.830277046754812, "cache_id": 1569, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.040351867675781, "actual_norm": 10.043149948120117, "relative_norm": 0.04999438103544965, "target_actual_norm": 10.043149948120117, "match_error": 0.0, "scalar": 0.17713174964968376, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -27.829204100824086, "first_log_odds": -27.75, "later_log_odds": -0.0792041008240858, "first_differing_log_odds": -27.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-j.npz", "array_sha256": "405e5402f3834230d6e36b477090baa27592294b5e03a8726cab31c1fc35c084", "seconds": 0.33785720774903893, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457698304, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-opposite", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.050505638122559, "actual_norm": 10.043195724487305, "relative_norm": 0.04999460890829387, "target_actual_norm": 10.043149948120117, "match_error": 4.57763671875e-05, "scalar": 10.050505770836025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1570, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-5.989677235643535e-05, -0.00013190258632624752], "logp": -0.00019179935868268288, "cache_id": 1571, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.050505638122559, "actual_norm": 10.043195724487305, "relative_norm": 0.04999460890829387, "target_actual_norm": 10.043149948120117, "match_error": 4.57763671875e-05, "scalar": 10.050505770836025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.625059896772356, -0.07322750727450408], "logp": -29.69828740404686, "cache_id": 1572, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.050505638122559, "actual_norm": 10.043195724487305, "relative_norm": 0.04999460890829387, "target_actual_norm": 10.043149948120117, "match_error": 4.57763671875e-05, "scalar": 10.050505770836025, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.698095604688177, "first_log_odds": -29.625, "later_log_odds": -0.07309560468817722, "first_differing_log_odds": -29.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-opposite.npz", "array_sha256": "8d61ba1e84e61c7fa6014ba79df388609413cef235fe4489e6f40b9484f1c5ce", "seconds": 0.3421942498534918, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457698304, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-unrelated", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.03299617767334, "actual_norm": 10.043238639831543, "relative_norm": 0.04999482253908532, "target_actual_norm": 10.043149948120117, "match_error": 8.869171142578125e-05, "scalar": 10.032997380309098, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1573, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-6.6376195707445e-05, -0.00019310032521560955], "logp": -0.00025947652092305455, "cache_id": 1574, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.03299617767334, "actual_norm": 10.043238639831543, "relative_norm": 0.04999482253908532, "target_actual_norm": 10.043149948120117, "match_error": 8.869171142578125e-05, "scalar": 10.032997380309098, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.062566376195708, -0.08480709724042584], "logp": -29.147373473436133, "cache_id": 1575, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.03299617767334, "actual_norm": 10.043238639831543, "relative_norm": 0.04999482253908532, "target_actual_norm": 10.043149948120117, "match_error": 8.869171142578125e-05, "scalar": 10.032997380309098, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.14711399691521, "first_log_odds": -29.0625, "later_log_odds": -0.0846139969152091, "first_differing_log_odds": -29.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-unrelated.npz", "array_sha256": "c0ebf8391ad5998bb5ccdfe1ebe6b5d1150340c89b53b30036be536a935808bd", "seconds": 0.34490716690197587, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457698304, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-ordinary", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.045477867126465, "actual_norm": 10.043229103088379, "relative_norm": 0.049994775065576105, "target_actual_norm": 10.043149948120117, "match_error": 7.915496826171875e-05, "scalar": 10.045477376401323, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1576, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-9.480286284294769e-05, -0.00042120215936118535], "logp": -0.000516005022204133, "cache_id": 1577, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.045477867126465, "actual_norm": 10.043229103088379, "relative_norm": 0.049994775065576105, "target_actual_norm": 10.043149948120117, "match_error": 7.915496826171875e-05, "scalar": 10.045477376401323, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.812594802862844, -0.07205925865077346], "logp": -28.88465406151362, "cache_id": 1578, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.045477867126465, "actual_norm": 10.043229103088379, "relative_norm": 0.049994775065576105, "target_actual_norm": 10.043149948120117, "match_error": 7.915496826171875e-05, "scalar": 10.045477376401323, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.884138056491416, "first_log_odds": -28.8125, "later_log_odds": -0.07163805649141608, "first_differing_log_odds": -28.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-ordinary.npz", "array_sha256": "d371de8a7d2d51950a46a0d713c36a3ef5f6ff79dc6e2e327b1d452093912b64", "seconds": 0.34255824983119965, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457731072, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-random0", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.04092788696289, "actual_norm": 10.042487144470215, "relative_norm": 0.0499910816265594, "target_actual_norm": 10.043149948120117, "match_error": -0.0006628036499023438, "scalar": 10.040927876674687, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1579, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.565261274900091e-05, -0.0003281992185314234], "logp": -0.0004038518312804243, "cache_id": 1580, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.04092788696289, "actual_norm": 10.042487144470215, "relative_norm": 0.0499910816265594, "target_actual_norm": 10.043149948120117, "match_error": -0.0006628036499023438, "scalar": 10.040927876674687, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.12507565261275, -0.08224577901365118], "logp": -29.2073214316264, "cache_id": 1581, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.04092788696289, "actual_norm": 10.042487144470215, "relative_norm": 0.0499910816265594, "target_actual_norm": 10.043149948120117, "match_error": -0.0006628036499023438, "scalar": 10.040927876674687, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.20691757979512, "first_log_odds": -29.125, "later_log_odds": -0.08191757979512104, "first_differing_log_odds": -29.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-random0.npz", "array_sha256": "bb266333e3e67b1eb55207e971b521ce1c5c4d2a7e36b8a1ed2a9639e2990057", "seconds": 0.343964375089854, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457731072, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-random1", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.039626121520996, "actual_norm": 10.04324722290039, "relative_norm": 0.04999486526524361, "target_actual_norm": 10.043149948120117, "match_error": 9.72747802734375e-05, "scalar": 10.039625283068744, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1582, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.505313596758814e-05, -0.00023780983408741587], "logp": -0.000312862970055004, "cache_id": 1583, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.039626121520996, "actual_norm": 10.04324722290039, "relative_norm": 0.04999486526524361, "target_actual_norm": 10.043149948120117, "match_error": 9.72747802734375e-05, "scalar": 10.039625283068744, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.18757505313597, -0.0767394175071721], "logp": -29.264314470643143, "cache_id": 1584, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.039626121520996, "actual_norm": 10.04324722290039, "relative_norm": 0.04999486526524361, "target_actual_norm": 10.043149948120117, "match_error": 9.72747802734375e-05, "scalar": 10.039625283068744, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.264001607673087, "first_log_odds": -29.1875, "later_log_odds": -0.07650160767308734, "first_differing_log_odds": -29.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-random1.npz", "array_sha256": "fc609cc126b175bd1e7eb1a2dfa5caf576cdab14e76a3aeba03cf8e5336e79ac", "seconds": 0.34397379076108336, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457731072, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-random2", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.032574653625488, "actual_norm": 10.043174743652344, "relative_norm": 0.0499945044665736, "target_actual_norm": 10.043149948120117, "match_error": 2.47955322265625e-05, "scalar": 10.032575952965999, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1585, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-5.8040618612811794e-05, -0.00029733791550841875], "logp": -0.00035537853412123053, "cache_id": 1586, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.032574653625488, "actual_norm": 10.043174743652344, "relative_norm": 0.0499945044665736, "target_actual_norm": 10.043149948120117, "match_error": 2.47955322265625e-05, "scalar": 10.032575952965999, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.31255804061861, -0.10030400088223945], "logp": -29.41286204150085, "cache_id": 1587, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 200.8855743408203, "strength_one_norm": 56.682960510253906, "target_shrink_scale": 0.17713174964968376, "requested_norm": 10.032574653625488, "actual_norm": 10.043174743652344, "relative_norm": 0.0499945044665736, "target_actual_norm": 10.043149948120117, "match_error": 2.47955322265625e-05, "scalar": 10.032575952965999, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.41250666296673, "first_log_odds": -29.3125, "later_log_odds": -0.10000666296673089, "first_differing_log_odds": -29.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-random2.npz", "array_sha256": "dfc83520b4ab1f925cbcb50457738c90a0798946691da98f23cb8925282e8f00", "seconds": 0.3532114173285663, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457731072, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-main-bridge", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": " Malaysia is not the correct answer. The currency used in Geneva, Switzerland, is the Swiss Franc (CHF). The three-letter ISO 4217 code for the Swiss Franc is CHF.<|im_end|>", "output_text": " Malaysia is not the correct answer. The currency used in Geneva, Switzerland, is the Swiss Franc (CHF). The three-letter ISO 4217 code for the Swiss Franc is CHF.", "output_ids": [27696, 374, 537, 279, 4396, 4226, 13, 576, 11413, 1483, 304, 44245, 11, 29121, 11, 374, 279, 29691, 9694, 320, 2149, 37, 568, 576, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 29691, 9694, 374, 6826, 37, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 940.3731079101562, 195.76878356933594, 5.381254196166992, 0.594903826713562, 279.708984375, 365.1130676269531, 275.82330322265625, 374.2879638671875, 120.57907104492188, 157.66217041015625, 263.9950256347656, 290.2080993652344, 168.15245056152344, 185.8621368408203, 56.06795120239258, 102.77131652832031, 124.67137145996094, 263.5443420410156, 187.17982482910156, 540.0057373046875, 191.18441772460938, 498.23760986328125, 468.95068359375, 275.8526916503906, 24.88668441772461, 1124.00390625]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 940.2966918945312, 195.7936553955078, 5.565983772277832, 0.43969622254371643, 279.7178955078125, 365.0741882324219, 275.793701171875, 374.28814697265625, 120.60929870605469, 157.6930389404297, 264.0498352050781, 290.2288513183594, 168.119140625, 185.86891174316406, 56.16145324707031, 102.75357818603516, 124.71063995361328, 263.5376281738281, 187.2751922607422, 539.9994506835938, 191.15805053710938, 498.21484375, 469.0677490234375, 275.8995056152344, 24.97280502319336, 1123.9820556640625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.8789054751396179, 0.2073861062526703, 0.005544726271182299, 0.00041264723404310644, 0.272185355424881, 0.3560711443424225, 0.24296091496944427, 0.36926883459091187, 0.11639204621315002, 0.1613263636827469, 0.2726978063583374, 0.2932742238044739, 0.16178594529628754, 0.1775488704442978, 0.04554618522524834, 0.10459241271018982, 0.14135801792144775, 0.29188820719718933, 0.22456805408000946, 0.5368294715881348, 0.2027733325958252, 0.43753477931022644, 0.4512317478656769, 0.29861730337142944, 0.024175282567739487, 1.0468865633010864]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1588, "cached": true, "scoring": {"normalized": "Malaysia is not the correct answer. The currency used in Geneva, Switzerland, is the Swiss Franc (CHF). The three-letter ISO 4217 code for the Swiss Franc is CHF.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF", "CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": " Malaysia is not the correct answer. The currency used in Geneva, Switzerland, is the Swiss Franc (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysia", "start": 1, "end": 9}, {"text": "Switzerland", "start": 66, "end": 77}, {"text": "Swiss", "start": 86, "end": 91}, {"text": "Swiss", "start": 144, "end": 149}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-24.155433307684543, -0.00015124641189109427], "logp": -24.155584554096436, "cache_id": 1589, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 940.3731079101562, 195.76878356933594, 5.381254196166992, 0.594903826713562, 279.708984375, 365.1130676269531, 275.82330322265625, 374.2879638671875, 120.57907104492188, 157.66217041015625, 263.9950256347656, 290.2080993652344, 168.15245056152344, 185.8621368408203, 56.06795120239258, 102.77131652832031, 124.67137145996094, 263.5443420410156, 187.17982482910156, 540.0057373046875, 191.18441772460938, 498.23760986328125, 468.95068359375, 275.8526916503906, 24.88668441772461, 1124.00390625]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 940.2966918945312, 195.7936553955078, 5.565983772277832, 0.43969622254371643, 279.7178955078125, 365.0741882324219, 275.793701171875, 374.28814697265625, 120.60929870605469, 157.6930389404297, 264.0498352050781, 290.2288513183594, 168.119140625, 185.86891174316406, 56.16145324707031, 102.75357818603516, 124.71063995361328, 263.5376281738281, 187.2751922607422, 539.9994506835938, 191.15805053710938, 498.21484375, 469.0677490234375, 275.8995056152344, 24.97280502319336, 1123.9820556640625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.8789054751396179, 0.2073861062526703, 0.005544726271182299, 0.00041264723404310644, 0.272185355424881, 0.3560711443424225, 0.24296091496944427, 0.36926883459091187, 0.11639204621315002, 0.1613263636827469, 0.2726978063583374, 0.2932742238044739, 0.16178594529628754, 0.1775488704442978, 0.04554618522524834, 0.10459241271018982, 0.14135801792144775, 0.29188820719718933, 0.22456805408000946, 0.5368294715881348, 0.2027733325958252, 0.43753477931022644, 0.4512317478656769, 0.29861730337142944, 0.024175282567739487, 1.0468865633010864]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-9.280433307684543, -0.02637507258837433], "logp": -9.306808380272917, "cache_id": 1590, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 940.3731079101562, 195.76878356933594, 5.381254196166992, 0.594903826713562, 279.708984375, 365.1130676269531, 275.82330322265625, 374.2879638671875, 120.57907104492188, 157.66217041015625, 263.9950256347656, 290.2080993652344, 168.15245056152344, 185.8621368408203, 56.06795120239258, 102.77131652832031, 124.67137145996094, 263.5443420410156, 187.17982482910156, 540.0057373046875, 191.18441772460938, 498.23760986328125, 468.95068359375, 275.8526916503906, 24.88668441772461, 1124.00390625]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 940.2966918945312, 195.7936553955078, 5.565983772277832, 0.43969622254371643, 279.7178955078125, 365.0741882324219, 275.793701171875, 374.28814697265625, 120.60929870605469, 157.6930389404297, 264.0498352050781, 290.2288513183594, 168.119140625, 185.86891174316406, 56.16145324707031, 102.75357818603516, 124.71063995361328, 263.5376281738281, 187.2751922607422, 539.9994506835938, 191.15805053710938, 498.21484375, 469.0677490234375, 275.8995056152344, 24.97280502319336, 1123.9820556640625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.8789054751396179, 0.2073861062526703, 0.005544726271182299, 0.00041264723404310644, 0.272185355424881, 0.3560711443424225, 0.24296091496944427, 0.36926883459091187, 0.11639204621315002, 0.1613263636827469, 0.2726978063583374, 0.2932742238044739, 0.16178594529628754, 0.1775488704442978, 0.04554618522524834, 0.10459241271018982, 0.14135801792144775, 0.29188820719718933, 0.22456805408000946, 0.5368294715881348, 0.2027733325958252, 0.43753477931022644, 0.4512317478656769, 0.29861730337142944, 0.024175282567739487, 1.0468865633010864]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 14.848776173823518, "first_log_odds": 14.875, "later_log_odds": -0.02622382617648178, "first_differing_log_odds": 14.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-main-bridge.npz", "array_sha256": "e09bdd555a1c9f42dc5bfeecd8cac0a1bfad23fcb7bd5fa2f18b8f57739aedb9", "seconds": 1.6467670830897987, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-baseline", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1591, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.250749963935113, -1.3695037828192403e-05], "logp": -7.250763658972941, "cache_id": 1592, "trace": []}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-33.875749963935114, -0.0010036549795716396], "logp": -33.876753618914684, "cache_id": 1593, "trace": []}, "log_odds": -26.62598995994174, "first_log_odds": -26.625, "later_log_odds": -0.0009899599417408922, "first_differing_log_odds": -26.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-baseline.npz", "array_sha256": "ca9b57819f88e30d5d36a92f60eff37cc20be11ec0f97b16466dc0b106c54caa", "seconds": 1.17711962480098, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-noop", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.944464683532715, "match_error": -9.944464683532715, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1594, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.250749963935113, -1.3695037828192403e-05], "logp": -7.250763658972941, "cache_id": 1595, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.944464683532715, "match_error": -9.944464683532715, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-33.875749963935114, -0.0010036549795716396], "logp": -33.876753618914684, "cache_id": 1596, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.944464683532715, "match_error": -9.944464683532715, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.62598995994174, "first_log_odds": -26.625, "later_log_odds": -0.0009899599417408922, "first_differing_log_odds": -26.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-noop.npz", "array_sha256": "1e181595631fccbb6f940d3f3ccb86195e26a23ced77f3b370eba369a0df4e97", "seconds": 1.1588227082975209, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-j", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937507629394531, "actual_norm": 9.944464683532715, "relative_norm": 0.0499997880618643, "target_actual_norm": 9.944464683532715, "match_error": 0.0, "scalar": 0.17347133903986478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1597, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-8.250280218239201, -3.126609692833462e-05], "logp": -8.250311484336128, "cache_id": 1598, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937507629394531, "actual_norm": 9.944464683532715, "relative_norm": 0.0499997880618643, "target_actual_norm": 9.944464683532715, "match_error": 0.0, "scalar": 0.17347133903986478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-33.6877802182392, -0.0008223079187556434], "logp": -33.68860252615795, "cache_id": 1599, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937507629394531, "actual_norm": 9.944464683532715, "relative_norm": 0.0499997880618643, "target_actual_norm": 9.944464683532715, "match_error": 0.0, "scalar": 0.17347133903986478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -25.43829104182182, "first_log_odds": -25.437499999999996, "later_log_odds": -0.000791041821823768, "first_differing_log_odds": -25.437499999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-j.npz", "array_sha256": "15557402caf881b9874c58c430603d0f8e35356670d1d74d2102f05c2dcd9ad9", "seconds": 1.1684182086028159, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-opposite", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937408447265625, "actual_norm": 9.944421768188477, "relative_norm": 0.049999572287744244, "target_actual_norm": 9.944464683532715, "match_error": -4.291534423828125e-05, "scalar": 9.937408750796749, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1600, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.250772000597618, -6.216342319058218e-06], "logp": -7.2507782169399375, "cache_id": 1601, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937408447265625, "actual_norm": 9.944421768188477, "relative_norm": 0.049999572287744244, "target_actual_norm": 9.944464683532715, "match_error": -4.291534423828125e-05, "scalar": 9.937408750796749, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-34.43827200059762, -0.0005833460983805889], "logp": -34.438855346696, "cache_id": 1602, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.937408447265625, "actual_norm": 9.944421768188477, "relative_norm": 0.049999572287744244, "target_actual_norm": 9.944464683532715, "match_error": -4.291534423828125e-05, "scalar": 9.937408750796749, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -27.188077129756063, "first_log_odds": -27.187500000000004, "later_log_odds": -0.0005771297560599464, "first_differing_log_odds": -27.187500000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-opposite.npz", "array_sha256": "bf6e10f3beee83635ecd1acaf7269f7ca02d587c8ea81d104528580cef3aae32", "seconds": 1.1472061667591333, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-unrelated", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.928645133972168, "actual_norm": 9.944458961486816, "relative_norm": 0.049999759291981624, "target_actual_norm": 9.944464683532715, "match_error": -5.7220458984375e-06, "scalar": 9.928645737560146, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1603, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.500596916012354, -1.165300845796737e-05], "logp": -7.5006085690208115, "cache_id": 1604, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.928645133972168, "actual_norm": 9.944458961486816, "relative_norm": 0.049999759291981624, "target_actual_norm": 9.944464683532715, "match_error": -5.7220458984375e-06, "scalar": 9.928645737560146, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-34.156846916012356, -0.0005898087989828389], "logp": -34.15743672481134, "cache_id": 1605, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.928645133972168, "actual_norm": 9.944458961486816, "relative_norm": 0.049999759291981624, "target_actual_norm": 9.944464683532715, "match_error": -5.7220458984375e-06, "scalar": 9.928645737560146, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.656828155790528, "first_log_odds": -26.65625, "later_log_odds": -0.000578155790528001, "first_differing_log_odds": -26.65625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-unrelated.npz", "array_sha256": "3000881c85e878af898c4356a1af26e9bbc65d5d83a0dd2aa37b1c112bc2f921", "seconds": 1.1589969578199089, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-ordinary", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.944844245910645, "actual_norm": 9.944463729858398, "relative_norm": 0.049999783266883854, "target_actual_norm": 9.944464683532715, "match_error": -9.5367431640625e-07, "scalar": 9.944844034755079, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1606, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-8.250285714984969, -1.6614573626583652e-05], "logp": -8.250302329558595, "cache_id": 1607, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.944844245910645, "actual_norm": 9.944463729858398, "relative_norm": 0.049999783266883854, "target_actual_norm": 9.944464683532715, "match_error": -9.5367431640625e-07, "scalar": 9.944844034755079, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-34.40653571498497, -0.0008590303605374316], "logp": -34.407394745345506, "cache_id": 1608, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.944844245910645, "actual_norm": 9.944463729858398, "relative_norm": 0.049999783266883854, "target_actual_norm": 9.944464683532715, "match_error": -9.5367431640625e-07, "scalar": 9.944844034755079, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.157092415786913, "first_log_odds": -26.15625, "later_log_odds": -0.000842415786912909, "first_differing_log_odds": -26.15625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-ordinary.npz", "array_sha256": "66f42778349685062fdab4ba658eb08ba444ff1b1462e22b2a76a11c153cbfbc", "seconds": 1.1508552497252822, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-random0", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.943289756774902, "actual_norm": 9.944451332092285, "relative_norm": 0.04999972093213806, "target_actual_norm": 9.944464683532715, "match_error": -1.33514404296875e-05, "scalar": 9.943288694743387, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1609, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.500602188736945, -1.2359396929583182e-05], "logp": -7.500614548133875, "cache_id": 1610, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.943289756774902, "actual_norm": 9.944451332092285, "relative_norm": 0.04999972093213806, "target_actual_norm": 9.944464683532715, "match_error": -1.33514404296875e-05, "scalar": 9.943288694743387, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-34.12560218873694, -0.0008064610880549327], "logp": -34.126408649824995, "cache_id": 1611, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.943289756774902, "actual_norm": 9.944451332092285, "relative_norm": 0.04999972093213806, "target_actual_norm": 9.944464683532715, "match_error": -1.33514404296875e-05, "scalar": 9.943288694743387, "random_seed": 20273608, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.62579410169112, "first_log_odds": -26.624999999999996, "later_log_odds": -0.0007941016911239274, "first_differing_log_odds": -26.624999999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-random0.npz", "array_sha256": "7144b6a4f246fa4a675f30d34322d1f2aafd16040cc2694be5585e9294730442", "seconds": 1.1638560420833528, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-random1", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.933029174804688, "actual_norm": 9.94392204284668, "relative_norm": 0.049997059717990705, "target_actual_norm": 9.944464683532715, "match_error": -0.0005426406860351562, "scalar": 9.933027244178447, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1612, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-7.250754531390894, -1.0506051607561788e-05], "logp": -7.250765037442501, "cache_id": 1613, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.933029174804688, "actual_norm": 9.94392204284668, "relative_norm": 0.049997059717990705, "target_actual_norm": 9.944464683532715, "match_error": -0.0005426406860351562, "scalar": 9.933027244178447, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-33.938254531390896, -0.0008428867284389], "logp": -33.93909741811934, "cache_id": 1614, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.933029174804688, "actual_norm": 9.94392204284668, "relative_norm": 0.049997059717990705, "target_actual_norm": 9.944464683532715, "match_error": -0.0005426406860351562, "scalar": 9.933027244178447, "random_seed": 20273609, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.688332380676837, "first_log_odds": -26.687500000000004, "later_log_odds": -0.000832380676833111, "first_differing_log_odds": -26.687500000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-random1.npz", "array_sha256": "c929a88e08d3607baf5530d53c980db735d6b6f15d92e003835d600ca3bcd3c9", "seconds": 1.167578041087836, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-random2", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 44245, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.938091278076172, "actual_norm": 9.944299697875977, "relative_norm": 0.049998958530247196, "target_actual_norm": 9.944464683532715, "match_error": -0.00016498565673828125, "scalar": 9.938091582997004, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1615, "cached": true, "scoring": {"normalized": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Geneva is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 35, "end": 40}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-8.250295083124326, -1.3285952889700777e-05], "logp": -8.250308369077215, "cache_id": 1616, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.938091278076172, "actual_norm": 9.944299697875977, "relative_norm": 0.049998958530247196, "target_actual_norm": 9.944464683532715, "match_error": -0.00016498565673828125, "scalar": 9.938091582997004, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-34.59404508312433, -0.0007504722461172649], "logp": -34.594795555370446, "cache_id": 1617, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 198.89013671875, "strength_one_norm": 57.286163330078125, "target_shrink_scale": 0.17347133903986478, "requested_norm": 9.938091278076172, "actual_norm": 9.944299697875977, "relative_norm": 0.049998958530247196, "target_actual_norm": 9.944464683532715, "match_error": -0.00016498565673828125, "scalar": 9.938091582997004, "random_seed": 20273610, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.34448718629323, "first_log_odds": -26.34375, "later_log_odds": -0.0007371862932288309, "first_differing_log_odds": -26.34375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-random2.npz", "array_sha256": "5e576c8d33e47da68b817a00be3b9b4393f81b4256ed83513309ef51fcf1aad7", "seconds": 1.1792488750070333, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-27-paraphrase-bridge", "case_id": "eval-27", "city": "Geneva", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": " Malaysia's currency is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**. \n\nIf you're asking about a different currency, please specify the country.<|im_end|>", "output_text": " Malaysia's currency is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**. \n\nIf you're asking about a different currency, please specify the country.", "output_ids": [27696, 594, 11413, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 4710, 2679, 498, 2299, 10161, 911, 264, 2155, 11413, 11, 4486, 13837, 279, 3146, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 966.1771850585938, 52.42839813232422, 174.22760009765625, 188.63670349121094, 404.9700622558594, 314.6979064941406, 235.3463897705078, 348.797607421875, 116.14828491210938, 167.16506958007812, 278.07843017578125, 293.0373840332031, 155.940673828125, 177.0815887451172, 238.28147888183594, 226.14791870117188, 536.0985107421875, 208.25262451171875, 505.51806640625, 477.93988037109375, 293.0133056640625, 33.443424224853516, 782.0338745117188]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 966.2929077148438, 52.43143844604492, 174.19271850585938, 188.64088439941406, 404.9332275390625, 314.7133483886719, 235.41517639160156, 348.7727966308594, 116.1317367553711, 167.22055053710938, 278.1412353515625, 293.05670166015625, 155.9923095703125, 177.1077880859375, 238.24549865722656, 226.1525115966797, 536.05126953125, 208.15740966796875, 505.4888610839844, 477.9674987792969, 293.06622314453125, 33.476722717285156, 782.010009765625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.9010006189346313, 0.05244068056344986, 0.157866433262825, 0.17950622737407684, 0.3753119111061096, 0.30624935030937195, 0.2032296508550644, 0.33378517627716064, 0.1103706881403923, 0.16890837252140045, 0.28381630778312683, 0.28698259592056274, 0.14681938290596008, 0.1853889524936676, 0.2698672115802765, 0.2604534327983856, 0.5237321257591248, 0.22127479314804077, 0.44087204337120056, 0.45320919156074524, 0.3087960481643677, 0.032153237611055374, 0.8001523017883301]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1618, "cached": true, "scoring": {"normalized": "Malaysia's currency is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**. \n\nIf you're asking about a different currency, please specify the country.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": " Malaysia's currency is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysia", "start": 1, "end": 9}, {"text": "Malaysian", "start": 28, "end": 37}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-17.738372715376546, -1.0736073766057546e-05], "logp": -17.73838345145031, "cache_id": 1619, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 966.1771850585938, 52.42839813232422, 174.22760009765625, 188.63670349121094, 404.9700622558594, 314.6979064941406, 235.3463897705078, 348.797607421875, 116.14828491210938, 167.16506958007812, 278.07843017578125, 293.0373840332031, 155.940673828125, 177.0815887451172, 238.28147888183594, 226.14791870117188, 536.0985107421875, 208.25262451171875, 505.51806640625, 477.93988037109375, 293.0133056640625, 33.443424224853516, 782.0338745117188]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 966.2929077148438, 52.43143844604492, 174.19271850585938, 188.64088439941406, 404.9332275390625, 314.7133483886719, 235.41517639160156, 348.7727966308594, 116.1317367553711, 167.22055053710938, 278.1412353515625, 293.05670166015625, 155.9923095703125, 177.1077880859375, 238.24549865722656, 226.1525115966797, 536.05126953125, 208.15740966796875, 505.4888610839844, 477.9674987792969, 293.06622314453125, 33.476722717285156, 782.010009765625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.9010006189346313, 0.05244068056344986, 0.157866433262825, 0.17950622737407684, 0.3753119111061096, 0.30624935030937195, 0.2032296508550644, 0.33378517627716064, 0.1103706881403923, 0.16890837252140045, 0.28381630778312683, 0.28698259592056274, 0.14681938290596008, 0.1853889524936676, 0.2698672115802765, 0.2604534327983856, 0.5237321257591248, 0.22127479314804077, 0.44087204337120056, 0.45320919156074524, 0.3087960481643677, 0.032153237611055374, 0.8001523017883301]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-7.863372715376545, -0.000634638500123051], "logp": -7.8640073538766675, "cache_id": 1620, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 966.1771850585938, 52.42839813232422, 174.22760009765625, 188.63670349121094, 404.9700622558594, 314.6979064941406, 235.3463897705078, 348.797607421875, 116.14828491210938, 167.16506958007812, 278.07843017578125, 293.0373840332031, 155.940673828125, 177.0815887451172, 238.28147888183594, 226.14791870117188, 536.0985107421875, 208.25262451171875, 505.51806640625, 477.93988037109375, 293.0133056640625, 33.443424224853516, 782.0338745117188]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 966.2929077148438, 52.43143844604492, 174.19271850585938, 188.64088439941406, 404.9332275390625, 314.7133483886719, 235.41517639160156, 348.7727966308594, 116.1317367553711, 167.22055053710938, 278.1412353515625, 293.05670166015625, 155.9923095703125, 177.1077880859375, 238.24549865722656, 226.1525115966797, 536.05126953125, 208.15740966796875, 505.4888610839844, 477.9674987792969, 293.06622314453125, 33.476722717285156, 782.010009765625]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.9010006189346313, 0.05244068056344986, 0.157866433262825, 0.17950622737407684, 0.3753119111061096, 0.30624935030937195, 0.2032296508550644, 0.33378517627716064, 0.1103706881403923, 0.16890837252140045, 0.28381630778312683, 0.28698259592056274, 0.14681938290596008, 0.1853889524936676, 0.2698672115802765, 0.2604534327983856, 0.5237321257591248, 0.22127479314804077, 0.44087204337120056, 0.45320919156074524, 0.3087960481643677, 0.032153237611055374, 0.8001523017883301]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 9.874376097573643, "first_log_odds": 9.875, "later_log_odds": -0.0006239024263567927, "first_differing_log_odds": 9.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-27-paraphrase-bridge.npz", "array_sha256": "061b0ab1fa76dd820dbf4b545b2cb9088cbe85079839ab2a019cd413a374e9c6", "seconds": 1.7197143747471273, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-baseline", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1621, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.580159899851741e-05, -0.0003213000546780802], "logp": -0.0003671016536765976, "cache_id": 1622, "trace": []}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.937545801598997, -0.04275845040753778], "logp": -28.980304252006533, "cache_id": 1623, "trace": []}, "log_odds": -28.979937150352857, "first_log_odds": -28.9375, "later_log_odds": -0.04243715035285689, "first_differing_log_odds": -28.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-baseline.npz", "array_sha256": "1a842e7db10f918dbb8268999c30a3ee071c7a7075dbf186bd809246734853a3", "seconds": 0.3833590834401548, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-noop", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.696088790893555, "match_error": -9.696088790893555, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1624, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.580159899851741e-05, -0.0003213000546780802], "logp": -0.0003671016536765976, "cache_id": 1625, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.696088790893555, "match_error": -9.696088790893555, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.937545801598997, -0.04275845040753778], "logp": -28.980304252006533, "cache_id": 1626, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.696088790893555, "match_error": -9.696088790893555, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.979937150352857, "first_log_odds": -28.9375, "later_log_odds": -0.04243715035285689, "first_differing_log_odds": -28.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-noop.npz", "array_sha256": "6d5107df0aafcc99cac91534bf2713cb6cc8894f3078f8ff78d4c102b6eed90f", "seconds": 0.3396180421113968, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-j", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.67686939239502, "actual_norm": 9.696088790893555, "relative_norm": 0.04999929774374705, "target_actual_norm": 9.696088790893555, "match_error": 0.0, "scalar": 0.18659041073491617, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1627, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-3.965530178212647e-05, -0.0008525910462719281], "logp": -0.0008922463480540546, "cache_id": 1628, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.67686939239502, "actual_norm": 9.696088790893555, "relative_norm": 0.04999929774374705, "target_actual_norm": 9.696088790893555, "match_error": 0.0, "scalar": 0.18659041073491617, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.625039655301784, -0.044597615625614276], "logp": -28.6696372709274, "cache_id": 1629, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.67686939239502, "actual_norm": 9.696088790893555, "relative_norm": 0.04999929774374705, "target_actual_norm": 9.696088790893555, "match_error": 0.0, "scalar": 0.18659041073491617, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.668745024579344, "first_log_odds": -28.625, "later_log_odds": -0.04374502457934426, "first_differing_log_odds": -28.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-j.npz", "array_sha256": "28be58d9619626b71bf1602c7bae64aed39aae1ae754d1f751c801c90c45e60f", "seconds": 0.33909441716969013, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-opposite", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696212768554688, "relative_norm": 0.0499999370526608, "target_actual_norm": 9.696088790893555, "match_error": 0.0001239776611328125, "scalar": 9.685436349594966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1630, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.115752605804405e-05, -0.0002217106169490001], "logp": -0.0002628681430070441, "cache_id": 1631, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696212768554688, "relative_norm": 0.0499999370526608, "target_actual_norm": 9.696088790893555, "match_error": 0.0001239776611328125, "scalar": 9.685436349594966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.50004115752606, -0.035744123688190084], "logp": -29.53578528121425, "cache_id": 1632, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696212768554688, "relative_norm": 0.0499999370526608, "target_actual_norm": 9.696088790893555, "match_error": 0.0001239776611328125, "scalar": 9.685436349594966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.53552241307124, "first_log_odds": -29.5, "later_log_odds": -0.03552241307124149, "first_differing_log_odds": -29.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-opposite.npz", "array_sha256": "4e7a8b5f5fec016c01cbdda56c16b1e36958f934f97f529d63e61e82f9a9aa9c", "seconds": 0.3405249170027673, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-unrelated", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.693944931030273, "actual_norm": 9.696125984191895, "relative_norm": 0.04999948953642117, "target_actual_norm": 9.696088790893555, "match_error": 3.719329833984375e-05, "scalar": 9.693943507576478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1633, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.277785720836033e-05, -0.00025263826638835], "logp": -0.0002954161235967103, "cache_id": 1634, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.693944931030273, "actual_norm": 9.696125984191895, "relative_norm": 0.04999948953642117, "target_actual_norm": 9.696088790893555, "match_error": 3.719329833984375e-05, "scalar": 9.693943507576478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.062542777857207, -0.038239708595501375], "logp": -29.100782486452708, "cache_id": 1635, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.693944931030273, "actual_norm": 9.696125984191895, "relative_norm": 0.04999948953642117, "target_actual_norm": 9.696088790893555, "match_error": 3.719329833984375e-05, "scalar": 9.693943507576478, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.100487070329113, "first_log_odds": -29.0625, "later_log_odds": -0.03798707032911253, "first_differing_log_odds": -29.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-unrelated.npz", "array_sha256": "9e70826282a4e35ab5a452cfb2c787bf2edefe259e26102b11fb1eae80aa66b2", "seconds": 0.3413309999741614, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-ordinary", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691872596740723, "actual_norm": 9.695829391479492, "relative_norm": 0.04999796011278904, "target_actual_norm": 9.696088790893555, "match_error": -0.0002593994140625, "scalar": 9.691872199546196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1636, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.319056100784367e-05, -0.0006613765592640944], "logp": -0.0007045671202719382, "cache_id": 1637, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691872596740723, "actual_norm": 9.695829391479492, "relative_norm": 0.04999796011278904, "target_actual_norm": 9.696088790893555, "match_error": -0.0002593994140625, "scalar": 9.691872199546196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.00004319056101, -0.04065160490205422], "logp": -29.04069479546306, "cache_id": 1638, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691872596740723, "actual_norm": 9.695829391479492, "relative_norm": 0.04999796011278904, "target_actual_norm": 9.696088790893555, "match_error": -0.0002593994140625, "scalar": 9.691872199546196, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.039990228342788, "first_log_odds": -29.0, "later_log_odds": -0.03999022834278776, "first_differing_log_odds": -29.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-ordinary.npz", "array_sha256": "bb78f2f4b653e51a6a765e81d22a200932e414851cb2713efcd1d116cc451212", "seconds": 0.3428518329747021, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-random0", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691205978393555, "actual_norm": 9.695343971252441, "relative_norm": 0.049995456972503655, "target_actual_norm": 9.696088790893555, "match_error": -0.0007448196411132812, "scalar": 9.691206421965035, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1639, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-5.3901419054734616e-05, -0.0005146406661716027], "logp": -0.0005685420852263373, "cache_id": 1640, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691205978393555, "actual_norm": 9.695343971252441, "relative_norm": 0.049995456972503655, "target_actual_norm": 9.696088790893555, "match_error": -0.0007448196411132812, "scalar": 9.691206421965035, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.937553901419054, -0.03968974089372559], "logp": -28.97724364231278, "cache_id": 1641, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.691205978393555, "actual_norm": 9.695343971252441, "relative_norm": 0.049995456972503655, "target_actual_norm": 9.696088790893555, "match_error": -0.0007448196411132812, "scalar": 9.691206421965035, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -28.976675100227553, "first_log_odds": -28.9375, "later_log_odds": -0.03917510022755266, "first_differing_log_odds": -28.9375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-random0.npz", "array_sha256": "4bd9f64b0fa192f4a0ece44e5e74c5b604debdc15ae11af319aab71131b6bcc1", "seconds": 0.3460201248526573, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-random1", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696135520935059, "relative_norm": 0.04999953871402992, "target_actual_norm": 9.696088790893555, "match_error": 4.673004150390625e-05, "scalar": 9.685436349594966, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1642, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-4.020307588387363e-05, -0.00031863002476021875], "logp": -0.00035883310064409237, "cache_id": 1643, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696135520935059, "relative_norm": 0.04999953871402992, "target_actual_norm": 9.696088790893555, "match_error": 4.673004150390625e-05, "scalar": 9.685436349594966, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.250040203075883, -0.041219076209740864], "logp": -29.291259279285626, "cache_id": 1644, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.685436248779297, "actual_norm": 9.696135520935059, "relative_norm": 0.04999953871402992, "target_actual_norm": 9.696088790893555, "match_error": 4.673004150390625e-05, "scalar": 9.685436349594966, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.29090044618498, "first_log_odds": -29.25, "later_log_odds": -0.04090044618498112, "first_differing_log_odds": -29.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-random1.npz", "array_sha256": "710f47974bb482266a3ba17fbfe32d79614f04986dab1f773593835cf3d8c233", "seconds": 0.3455820418894291, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-random2", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "CHF<|im_end|>", "output_text": "CHF", "output_ids": [2149, 37, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.6975679397583, "actual_norm": 9.696107864379883, "relative_norm": 0.049999396098964546, "target_actual_norm": 9.696088790893555, "match_error": 1.9073486328125e-05, "scalar": 9.69756829662947, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1645, "cached": true, "scoring": {"normalized": "CHF", "bare_format": true, "malformed": false, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-3.986946771327529e-05, -0.00025053599749352384], "logp": -0.0002904054652067991, "cache_id": 1646, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.6975679397583, "actual_norm": 9.696107864379883, "relative_norm": 0.049999396098964546, "target_actual_norm": 9.696088790893555, "match_error": 1.9073486328125e-05, "scalar": 9.69756829662947, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.062539869467713, -0.03984132574922338], "logp": -29.102381195216935, "cache_id": 1647, "trace": [{"sequence_length": 36, "positions": [10], "layer": 13, "residual_norm": 193.92449951171875, "strength_one_norm": 51.8615608215332, "target_shrink_scale": 0.18659041073491617, "requested_norm": 9.6975679397583, "actual_norm": 9.696107864379883, "relative_norm": 0.049999396098964546, "target_actual_norm": 9.696088790893555, "match_error": 1.9073486328125e-05, "scalar": 9.69756829662947, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -29.102090789751728, "first_log_odds": -29.0625, "later_log_odds": -0.039590789751727584, "first_differing_log_odds": -29.0625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-random2.npz", "array_sha256": "2a35ae917e1ef35e89f61a34785f0d4e49788e688af1d2c5bd30a6804bc1bb31", "seconds": 0.3428995842114091, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-main-bridge", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": " Malaysia's currency is the Malaysian Ringgit (MYR). The three-letter ISO 4217 code for the Malaysian Ringgit is MYR.<|im_end|>", "output_text": " Malaysia's currency is the Malaysian Ringgit (MYR). The three-letter ISO 4217 code for the Malaysian Ringgit is MYR.", "output_ids": [27696, 594, 11413, 374, 279, 65431, 21525, 12882, 320, 19159, 49, 568, 576, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 369, 279, 65431, 21525, 12882, 374, 18224, 49, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 206.3304901123047, 431.5578918457031, 732.9093017578125, 198.07630920410156, 5.179811477661133, 14.766291618347168, 285.3839416503906, 365.91436767578125, 249.61968994140625, 347.0401306152344, 104.77130889892578, 140.75917053222656, 263.3865661621094, 271.8077087402344, 153.77415466308594, 175.10589599609375, 43.765830993652344, 96.66903686523438, 105.65030670166016, 260.83135986328125, 201.07293701171875, 551.5863037109375, 184.63612365722656, 494.8688659667969, 466.59423828125, 327.9114685058594, 32.798648834228516, 1187.9698486328125]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 206.27700805664062, 431.53021240234375, 732.857177734375, 198.10714721679688, 5.417584419250488, 14.844573974609375, 285.38525390625, 365.87335205078125, 249.6857452392578, 347.1221618652344, 104.81263732910156, 140.78726196289062, 263.39801025390625, 271.8924560546875, 153.7918701171875, 175.1498260498047, 43.82722473144531, 96.69086456298828, 105.65154266357422, 260.8495178222656, 201.09242248535156, 551.5197143554688, 184.64585876464844, 494.8351745605469, 466.5820617675781, 327.9478454589844, 32.7850341796875, 1187.8890380859375]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.18220293521881104, 0.32499897480010986, 0.7321792244911194, 0.20935295522212982, 0.005190431606024504, 0.013943713158369064, 0.2782176733016968, 0.3581385612487793, 0.21943171322345734, 0.34255048632621765, 0.10057513415813446, 0.14374832808971405, 0.2724878787994385, 0.2753724157810211, 0.1477299928665161, 0.1663162112236023, 0.03525124490261078, 0.09695946425199509, 0.11959879845380783, 0.28989091515541077, 0.2418805956840515, 0.5491088032722473, 0.19573372602462769, 0.4378170371055603, 0.45271801948547363, 0.3535531461238861, 0.0317453034222126, 1.1210908889770508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1648, "cached": true, "scoring": {"normalized": "Malaysia's currency is the Malaysian Ringgit (MYR). The three-letter ISO 4217 code for the Malaysian Ringgit is MYR.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR", "MYR"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": " Malaysia's currency is the Malaysian Ringgit (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysia", "start": 1, "end": 9}, {"text": "Malaysian", "start": 28, "end": 37}, {"text": "Malaysian", "start": 92, "end": 101}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-24.95133976915037, -0.0003313897819909852], "logp": -24.95167115893236, "cache_id": 1649, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 206.3304901123047, 431.5578918457031, 732.9093017578125, 198.07630920410156, 5.179811477661133, 14.766291618347168, 285.3839416503906, 365.91436767578125, 249.61968994140625, 347.0401306152344, 104.77130889892578, 140.75917053222656, 263.3865661621094, 271.8077087402344, 153.77415466308594, 175.10589599609375, 43.765830993652344, 96.66903686523438, 105.65030670166016, 260.83135986328125, 201.07293701171875, 551.5863037109375, 184.63612365722656, 494.8688659667969, 466.59423828125, 327.9114685058594, 32.798648834228516, 1187.9698486328125]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 206.27700805664062, 431.53021240234375, 732.857177734375, 198.10714721679688, 5.417584419250488, 14.844573974609375, 285.38525390625, 365.87335205078125, 249.6857452392578, 347.1221618652344, 104.81263732910156, 140.78726196289062, 263.39801025390625, 271.8924560546875, 153.7918701171875, 175.1498260498047, 43.82722473144531, 96.69086456298828, 105.65154266357422, 260.8495178222656, 201.09242248535156, 551.5197143554688, 184.64585876464844, 494.8351745605469, 466.5820617675781, 327.9478454589844, 32.7850341796875, 1187.8890380859375]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.18220293521881104, 0.32499897480010986, 0.7321792244911194, 0.20935295522212982, 0.005190431606024504, 0.013943713158369064, 0.2782176733016968, 0.3581385612487793, 0.21943171322345734, 0.34255048632621765, 0.10057513415813446, 0.14374832808971405, 0.2724878787994385, 0.2753724157810211, 0.1477299928665161, 0.1663162112236023, 0.03525124490261078, 0.09695946425199509, 0.11959879845380783, 0.28989091515541077, 0.2418805956840515, 0.5491088032722473, 0.19573372602462769, 0.4378170371055603, 0.45271801948547363, 0.3535531461238861, 0.0317453034222126, 1.1210908889770508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-8.170089769150369, -0.02758434295561873], "logp": -8.197674112105988, "cache_id": 1650, "trace": [{"sequence_length": 36, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 1.7481566667556763, 15.117977142333984, 61.38600540161133, 52.37228775024414, 19.523447036743164, 206.3304901123047, 431.5578918457031, 732.9093017578125, 198.07630920410156, 5.179811477661133, 14.766291618347168, 285.3839416503906, 365.91436767578125, 249.61968994140625, 347.0401306152344, 104.77130889892578, 140.75917053222656, 263.3865661621094, 271.8077087402344, 153.77415466308594, 175.10589599609375, 43.765830993652344, 96.66903686523438, 105.65030670166016, 260.83135986328125, 201.07293701171875, 551.5863037109375, 184.63612365722656, 494.8688659667969, 466.59423828125, 327.9114685058594, 32.798648834228516, 1187.9698486328125]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 1.7735066413879395, 15.14094066619873, 61.43309783935547, 52.36652755737305, 19.543315887451172, 206.27700805664062, 431.53021240234375, 732.857177734375, 198.10714721679688, 5.417584419250488, 14.844573974609375, 285.38525390625, 365.87335205078125, 249.6857452392578, 347.1221618652344, 104.81263732910156, 140.78726196289062, 263.39801025390625, 271.8924560546875, 153.7918701171875, 175.1498260498047, 43.82722473144531, 96.69086456298828, 105.65154266357422, 260.8495178222656, 201.09242248535156, 551.5197143554688, 184.64585876464844, 494.8351745605469, 466.5820617675781, 327.9478454589844, 32.7850341796875, 1187.8890380859375]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.001628016703762114, 0.013284378685057163, 0.049979083240032196, 0.04482157155871391, 0.017221584916114807, 0.18220293521881104, 0.32499897480010986, 0.7321792244911194, 0.20935295522212982, 0.005190431606024504, 0.013943713158369064, 0.2782176733016968, 0.3581385612487793, 0.21943171322345734, 0.34255048632621765, 0.10057513415813446, 0.14374832808971405, 0.2724878787994385, 0.2753724157810211, 0.1477299928665161, 0.1663162112236023, 0.03525124490261078, 0.09695946425199509, 0.11959879845380783, 0.28989091515541077, 0.2418805956840515, 0.5491088032722473, 0.19573372602462769, 0.4378170371055603, 0.45271801948547363, 0.3535531461238861, 0.0317453034222126, 1.1210908889770508]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 16.75399704682637, "first_log_odds": 16.78125, "later_log_odds": -0.027252953173629635, "first_differing_log_odds": 16.78125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-main-bridge.npz", "array_sha256": "052ba3cd218c6ef4050e91efbaf4e47913a231145791a3f518daacaf586853fb", "seconds": 1.315476707648486, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-baseline", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1651, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-10.000055743528764, -2.4391732685845257e-05], "logp": -10.000080135261449, "cache_id": 1652, "trace": []}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-36.15630574352876, -0.0017614138375822716], "logp": -36.15806715736635, "cache_id": 1653, "trace": []}, "log_odds": -26.157987022104898, "first_log_odds": -26.15625, "later_log_odds": -0.0017370221048977896, "first_differing_log_odds": -26.15625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-baseline.npz", "array_sha256": "102302fef514a2c1ac36ef1ec0a856ad6534d30ff62d149b3695f129b853f83c", "seconds": 1.2756872498430312, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-noop", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.6148099899292, "match_error": -9.6148099899292, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1654, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-10.000055743528764, -2.4391732685845257e-05], "logp": -10.000080135261449, "cache_id": 1655, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.6148099899292, "match_error": -9.6148099899292, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-36.15630574352876, -0.0017614138375822716], "logp": -36.15806715736635, "cache_id": 1656, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.6148099899292, "match_error": -9.6148099899292, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.157987022104898, "first_log_odds": -26.15625, "later_log_odds": -0.0017370221048977896, "first_differing_log_odds": -26.15625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-noop.npz", "array_sha256": "a071b8125bfa10501f70690379d58dad2e1a076779dfc9b18a02ca80fd4b1650", "seconds": 1.2496642908081412, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-j", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.609280586242676, "actual_norm": 9.6148099899292, "relative_norm": 0.04999899127781633, "target_actual_norm": 9.6148099899292, "match_error": 0.0, "scalar": 0.19505124536859966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1657, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-10.250040973958537, -6.330094309294695e-05], "logp": -10.25010427490163, "cache_id": 1658, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.609280586242676, "actual_norm": 9.6148099899292, "relative_norm": 0.04999899127781633, "target_actual_norm": 9.6148099899292, "match_error": 0.0, "scalar": 0.19505124536859966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-35.968790973958534, -0.0017180605619851804], "logp": -35.97050903452052, "cache_id": 1659, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.609280586242676, "actual_norm": 9.6148099899292, "relative_norm": 0.04999899127781633, "target_actual_norm": 9.6148099899292, "match_error": 0.0, "scalar": 0.19505124536859966, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -25.720404759618887, "first_log_odds": -25.718749999999996, "later_log_odds": -0.001654759618890722, "first_differing_log_odds": -25.718749999999996, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-j.npz", "array_sha256": "afe7ec05a70ef230c2c7b8b3dabb22179535035ed47c10e19291ab5e85b8072b", "seconds": 1.2580517092719674, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-opposite", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.616899490356445, "actual_norm": 9.611774444580078, "relative_norm": 0.04998320581709552, "target_actual_norm": 9.6148099899292, "match_error": -0.0030355453491210938, "scalar": 9.61690061252375, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1660, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-9.500091816761298, -1.5591430484473596e-05], "logp": -9.500107408191782, "cache_id": 1661, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.616899490356445, "actual_norm": 9.611774444580078, "relative_norm": 0.04998320581709552, "target_actual_norm": 9.6148099899292, "match_error": -0.0030355453491210938, "scalar": 9.61690061252375, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-36.3125918167613, -0.0014185631691254698], "logp": -36.31401037993042, "cache_id": 1662, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.616899490356445, "actual_norm": 9.611774444580078, "relative_norm": 0.04998320581709552, "target_actual_norm": 9.6148099899292, "match_error": -0.0030355453491210938, "scalar": 9.61690061252375, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.813902971738642, "first_log_odds": -26.8125, "later_log_odds": -0.0014029717386421225, "first_differing_log_odds": -26.8125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-opposite.npz", "array_sha256": "b5fee34c37d3643488c4b50dce699620751f48016d29cad2c6f3b399125b1e3d", "seconds": 1.2636372498236597, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-unrelated", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60615348815918, "actual_norm": 9.614950180053711, "relative_norm": 0.0499997202953237, "target_actual_norm": 9.6148099899292, "match_error": 0.00014019012451171875, "scalar": 9.60615407883597, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1663, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-10.000055721547712, -2.075697541021225e-05], "logp": -10.000076478523122, "cache_id": 1664, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60615348815918, "actual_norm": 9.614950180053711, "relative_norm": 0.0499997202953237, "target_actual_norm": 9.6148099899292, "match_error": 0.00014019012451171875, "scalar": 9.60615407883597, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-36.40630572154771, -0.0014627443514193025], "logp": -36.40776846589913, "cache_id": 1665, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60615348815918, "actual_norm": 9.614950180053711, "relative_norm": 0.0499997202953237, "target_actual_norm": 9.6148099899292, "match_error": 0.00014019012451171875, "scalar": 9.60615407883597, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.407691987376012, "first_log_odds": -26.40625, "later_log_odds": -0.0014419873760118662, "first_differing_log_odds": -26.40625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-unrelated.npz", "array_sha256": "83aecfd862d32024c0f622442bf7ef7ea506d64fce6f5bfe06c3240110408a5f", "seconds": 1.2354199159890413, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-ordinary", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60483455657959, "actual_norm": 9.614827156066895, "relative_norm": 0.049999080545266215, "target_actual_norm": 9.6148099899292, "match_error": 1.71661376953125e-05, "scalar": 9.60483368561836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1666, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-9.75007157101845, -4.000253993800684e-05], "logp": -9.750111573558389, "cache_id": 1667, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60483455657959, "actual_norm": 9.614827156066895, "relative_norm": 0.049999080545266215, "target_actual_norm": 9.6148099899292, "match_error": 1.71661376953125e-05, "scalar": 9.60483368561836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-35.93757157101845, -0.0015289072424961821], "logp": -35.93910047826095, "cache_id": 1668, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.60483455657959, "actual_norm": 9.614827156066895, "relative_norm": 0.049999080545266215, "target_actual_norm": 9.6148099899292, "match_error": 1.71661376953125e-05, "scalar": 9.60483368561836, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.18898890470256, "first_log_odds": -26.1875, "later_log_odds": -0.001488904702558358, "first_differing_log_odds": -26.1875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-ordinary.npz", "array_sha256": "bc97927797af7cf2832ce9905d400c612bf78bb9b05ecb4fccd45e00063b656d", "seconds": 1.250248999800533, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-random0", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.608501434326172, "actual_norm": 9.614801406860352, "relative_norm": 0.049998946644091394, "target_actual_norm": 9.6148099899292, "match_error": -8.58306884765625e-06, "scalar": 9.608501444556168, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1669, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-9.500088152970882, -2.652358766447164e-05], "logp": -9.500114676558546, "cache_id": 1670, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.608501434326172, "actual_norm": 9.614801406860352, "relative_norm": 0.049998946644091394, "target_actual_norm": 9.6148099899292, "match_error": -8.58306884765625e-06, "scalar": 9.608501444556168, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-35.93758815297088, -0.0017627713527742692], "logp": -35.939350924323655, "cache_id": 1671, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.608501434326172, "actual_norm": 9.614801406860352, "relative_norm": 0.049998946644091394, "target_actual_norm": 9.6148099899292, "match_error": -8.58306884765625e-06, "scalar": 9.608501444556168, "random_seed": 20273708, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.43923624776511, "first_log_odds": -26.4375, "later_log_odds": -0.0017362477651090558, "first_differing_log_odds": -26.4375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-random0.npz", "array_sha256": "cd7ed5464b09ce3b6689746b5d69b0f5fd9ee3ddc7e02b8f8e362eba2b02da6e", "seconds": 1.267201166599989, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-random1", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.59492015838623, "actual_norm": 9.614847183227539, "relative_norm": 0.04999918469062441, "target_actual_norm": 9.6148099899292, "match_error": 3.719329833984375e-05, "scalar": 9.594921567088932, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1672, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-10.000055741486268, -2.3796209896478728e-05], "logp": -10.000079537696164, "cache_id": 1673, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.59492015838623, "actual_norm": 9.614847183227539, "relative_norm": 0.04999918469062441, "target_actual_norm": 9.6148099899292, "match_error": 3.719329833984375e-05, "scalar": 9.594921567088932, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-36.18755574148627, -0.0014550179481311778], "logp": -36.1890107594344, "cache_id": 1674, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.59492015838623, "actual_norm": 9.614847183227539, "relative_norm": 0.04999918469062441, "target_actual_norm": 9.6148099899292, "match_error": 3.719329833984375e-05, "scalar": 9.594921567088932, "random_seed": 20273709, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.188931221738237, "first_log_odds": -26.187500000000004, "later_log_odds": -0.0014312217382332904, "first_differing_log_odds": -26.187500000000004, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-random1.npz", "array_sha256": "368361b0bdb2bd922bb16952f76e1a2b72c4dee1dfa78ae4797a44087c29c964", "seconds": 1.265123459044844, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-random2", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.614663124084473, "actual_norm": 9.614816665649414, "relative_norm": 0.04999902599293573, "target_actual_norm": 9.6148099899292, "match_error": 6.67572021484375e-06, "scalar": 9.614663279571687, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1675, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-9.750071578695183, -2.8173971067273036e-05], "logp": -9.75009975266625, "cache_id": 1676, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.614663124084473, "actual_norm": 9.614816665649414, "relative_norm": 0.04999902599293573, "target_actual_norm": 9.6148099899292, "match_error": 6.67572021484375e-06, "scalar": 9.614663279571687, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-35.968821578695184, -0.0017843791402241385], "logp": -35.97060595783541, "cache_id": 1677, "trace": [{"sequence_length": 33, "positions": [10], "layer": 13, "residual_norm": 192.30007934570312, "strength_one_norm": 49.265419006347656, "target_shrink_scale": 0.19505124536859966, "requested_norm": 9.614663124084473, "actual_norm": 9.614816665649414, "relative_norm": 0.04999902599293573, "target_actual_norm": 9.6148099899292, "match_error": 6.67572021484375e-06, "scalar": 9.614663279571687, "random_seed": 20273710, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -26.22050620516916, "first_log_odds": -26.21875, "later_log_odds": -0.0017562051691584202, "first_differing_log_odds": -26.21875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-random2.npz", "array_sha256": "d5ef4e2f7fb84534136ad9976820a6cb22496e58befee368dfb19a1f65cf036f", "seconds": 1.2668262496590614, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-28-paraphrase-bridge", "case_id": "eval-28", "city": "Lausanne", "country": "Switzerland", "code": "CHF", "target_country": "Malaysia", "target_code": "MYR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.<|im_end|>", "output_text": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "output_ids": [785, 11413, 1483, 304, 4929, 355, 20368, 374, 279, 29691, 9694, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 2149, 37, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 206.94679260253906, 422.2660827636719, 742.12451171875, 90.60746765136719, 173.12408447265625, 206.9303436279297, 326.7720642089844, 314.398193359375, 207.02001953125, 334.8041076660156, 92.75170135498047, 153.74859619140625, 275.3911437988281, 274.1312561035156, 141.66668701171875, 144.49171447753906, 249.6461639404297, 276.0381774902344, 543.2175903320312, 212.5443572998047, 504.72015380859375, 481.9987487792969, 300.9234619140625, 31.05414581298828, 759.6099243164062]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 206.94659423828125, 422.21453857421875, 742.103515625, 90.65884399414062, 173.11328125, 206.9882049560547, 326.79254150390625, 314.37713623046875, 206.92869567871094, 334.8365478515625, 92.76705169677734, 153.738525390625, 275.405517578125, 274.1067810058594, 141.6464080810547, 144.48983764648438, 249.69070434570312, 276.0082092285156, 543.2305297851562, 212.5272216796875, 504.6416931152344, 482.07147216796875, 300.8351135253906, 31.1168212890625, 759.611328125]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.1832728534936905, 0.31798022985458374, 0.7412429451942444, 0.0911221131682396, 0.1582278609275818, 0.19607257843017578, 0.30180346965789795, 0.31109511852264404, 0.17890958487987518, 0.3205624520778656, 0.08750424534082413, 0.15635967254638672, 0.2829256057739258, 0.27019959688186646, 0.13379579782485962, 0.152537003159523, 0.2811432182788849, 0.3167048692703247, 0.5321471691131592, 0.22578825056552887, 0.4409532845020294, 0.45736849308013916, 0.3199889659881592, 0.029980620369315147, 0.7818431258201599]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1678, "cached": true, "scoring": {"normalized": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **CHF**.", "bare_format": false, "malformed": true, "first_code": "CHF", "all_codes": ["CHF"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Lausanne is the Swiss Franc, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Swiss", "start": 37, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [2149, 37], "tokens": ["CH", "F"], "log_probs": [-18.72982206427809, -2.4596974921693017e-05], "logp": -18.72984666125301, "cache_id": 1679, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 206.94679260253906, 422.2660827636719, 742.12451171875, 90.60746765136719, 173.12408447265625, 206.9303436279297, 326.7720642089844, 314.398193359375, 207.02001953125, 334.8041076660156, 92.75170135498047, 153.74859619140625, 275.3911437988281, 274.1312561035156, 141.66668701171875, 144.49171447753906, 249.6461639404297, 276.0381774902344, 543.2175903320312, 212.5443572998047, 504.72015380859375, 481.9987487792969, 300.9234619140625, 31.05414581298828, 759.6099243164062]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 206.94659423828125, 422.21453857421875, 742.103515625, 90.65884399414062, 173.11328125, 206.9882049560547, 326.79254150390625, 314.37713623046875, 206.92869567871094, 334.8365478515625, 92.76705169677734, 153.738525390625, 275.405517578125, 274.1067810058594, 141.6464080810547, 144.48983764648438, 249.69070434570312, 276.0082092285156, 543.2305297851562, 212.5272216796875, 504.6416931152344, 482.07147216796875, 300.8351135253906, 31.1168212890625, 759.611328125]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.1832728534936905, 0.31798022985458374, 0.7412429451942444, 0.0911221131682396, 0.1582278609275818, 0.19607257843017578, 0.30180346965789795, 0.31109511852264404, 0.17890958487987518, 0.3205624520778656, 0.08750424534082413, 0.15635967254638672, 0.2829256057739258, 0.27019959688186646, 0.13379579782485962, 0.152537003159523, 0.2811432182788849, 0.3167048692703247, 0.5321471691131592, 0.22578825056552887, 0.4409532845020294, 0.45736849308013916, 0.3199889659881592, 0.029980620369315147, 0.7818431258201599]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-8.417322064278089, -0.0016622052037166338], "logp": -8.418984269481806, "cache_id": 1680, "trace": [{"sequence_length": 33, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32], "requested_norm": [[370.022705078125, 137.21954345703125, 129.51885986328125, 193.40289306640625, 67.1755142211914, 3.9966135025024414, 25.76493263244629, 37.55556106567383, 206.94679260253906, 422.2660827636719, 742.12451171875, 90.60746765136719, 173.12408447265625, 206.9303436279297, 326.7720642089844, 314.398193359375, 207.02001953125, 334.8041076660156, 92.75170135498047, 153.74859619140625, 275.3911437988281, 274.1312561035156, 141.66668701171875, 144.49171447753906, 249.6461639404297, 276.0381774902344, 543.2175903320312, 212.5443572998047, 504.72015380859375, 481.9987487792969, 300.9234619140625, 31.05414581298828, 759.6099243164062]], "actual_norm": [[370.08306884765625, 137.2350616455078, 129.52952575683594, 193.430908203125, 67.21753692626953, 4.16085958480835, 25.78704261779785, 37.521461486816406, 206.94659423828125, 422.21453857421875, 742.103515625, 90.65884399414062, 173.11328125, 206.9882049560547, 326.79254150390625, 314.37713623046875, 206.92869567871094, 334.8365478515625, 92.76705169677734, 153.738525390625, 275.405517578125, 274.1067810058594, 141.6464080810547, 144.48983764648438, 249.69070434570312, 276.0082092285156, 543.2305297851562, 212.5272216796875, 504.6416931152344, 482.07147216796875, 300.8351135253906, 31.1168212890625, 759.611328125]], "relative_norm": [[0.021599851548671722, 0.14557604491710663, 0.14330004155635834, 0.1686515212059021, 0.06118645891547203, 0.003614688292145729, 0.022934703156352043, 0.03333054855465889, 0.1832728534936905, 0.31798022985458374, 0.7412429451942444, 0.0911221131682396, 0.1582278609275818, 0.19607257843017578, 0.30180346965789795, 0.31109511852264404, 0.17890958487987518, 0.3205624520778656, 0.08750424534082413, 0.15635967254638672, 0.2829256057739258, 0.27019959688186646, 0.13379579782485962, 0.152537003159523, 0.2811432182788849, 0.3167048692703247, 0.5321471691131592, 0.22578825056552887, 0.4409532845020294, 0.45736849308013916, 0.3199889659881592, 0.029980620369315147, 0.7818431258201599]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 10.310862391771204, "first_log_odds": 10.3125, "later_log_odds": -0.0016376082287958837, "first_differing_log_odds": 10.3125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-28-paraphrase-bridge.npz", "array_sha256": "b73c326b712f80701f04251a4e86dd7b799fd5e206e0eaf53037ec4c6de22d72", "seconds": 1.2498333752155304, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-baseline", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1681, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.871168835803804, -0.0031280765471923882], "logp": -4.874296912350997, "cache_id": 1682, "trace": []}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.996168835803804, -1.1792015738728246e-08], "logp": -4.996168847595819, "cache_id": 1683, "trace": []}, "log_odds": -0.12187193524482254, "first_log_odds": -0.125, "later_log_odds": 0.003128064755177462, "first_differing_log_odds": -0.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-baseline.npz", "array_sha256": "2d4e121585183ada5a390551dcfd4e4f44578ba1adeb78ca1851f1bafaa6d7d6", "seconds": 0.35693833278492093, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457763840, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-noop", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.739886283874512, "match_error": -9.739886283874512, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1684, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.871168835803804, -0.0031280765471923882], "logp": -4.874296912350997, "cache_id": 1685, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.739886283874512, "match_error": -9.739886283874512, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.996168835803804, -1.1792015738728246e-08], "logp": -4.996168847595819, "cache_id": 1686, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.739886283874512, "match_error": -9.739886283874512, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.12187193524482254, "first_log_odds": -0.125, "later_log_odds": 0.003128064755177462, "first_differing_log_odds": -0.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-noop.npz", "array_sha256": "42a8180b6d5021e66a62d4fe9d9034c68e9d148e49f5587d531160df20a3b5f8", "seconds": 0.3172564576379955, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457796608, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-j", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741333961486816, "actual_norm": 9.739886283874512, "relative_norm": 0.049992570386604616, "target_actual_norm": 9.739886283874512, "match_error": 0.0, "scalar": 0.2720558602390303, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1687, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-5.698793180894826, -0.005333527375663778], "logp": -5.704126708270489, "cache_id": 1688, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741333961486816, "actual_norm": 9.739886283874512, "relative_norm": 0.049992570386604616, "target_actual_norm": 9.739886283874512, "match_error": 0.0, "scalar": 0.2720558602390303, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-1.9487931808948256, -1.0686565353265826e-08], "logp": -1.948793191581391, "cache_id": 1689, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741333961486816, "actual_norm": 9.739886283874512, "relative_norm": 0.049992570386604616, "target_actual_norm": 9.739886283874512, "match_error": 0.0, "scalar": 0.2720558602390303, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 3.755333516689098, "first_log_odds": 3.75, "later_log_odds": 0.0053335166890979835, "first_differing_log_odds": 3.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-j.npz", "array_sha256": "1a86f987684ce0bedfee95bc20b3724c209264c4c7ecc1426eb3b3151ef3d8bd", "seconds": 0.316619707737118, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457796608, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-opposite", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741223335266113, "actual_norm": 9.739531517028809, "relative_norm": 0.04999074945091936, "target_actual_norm": 9.739886283874512, "match_error": -0.000354766845703125, "scalar": 9.741223853707197, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1690, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.705551884283757, -0.004019311634008805], "logp": -4.709571195917766, "cache_id": 1691, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741223335266113, "actual_norm": 9.739531517028809, "relative_norm": 0.04999074945091936, "target_actual_norm": 9.739886283874512, "match_error": -0.000354766845703125, "scalar": 9.741223853707197, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-7.705551884283757, -1.1231617350823606e-08], "logp": -7.705551895515375, "cache_id": 1692, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.741223335266113, "actual_norm": 9.739531517028809, "relative_norm": 0.04999074945091936, "target_actual_norm": 9.739886283874512, "match_error": -0.000354766845703125, "scalar": 9.741223853707197, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -2.9959806995976086, "first_log_odds": -3.0, "later_log_odds": 0.004019300402391401, "first_differing_log_odds": -3.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-opposite.npz", "array_sha256": "e887cbea4818bbcb315b55d4db4d5bf0d93552694e8750fdceb0a288f1283b6f", "seconds": 0.3174588750116527, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457796608, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-unrelated", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.739886283874512, "actual_norm": 9.739957809448242, "relative_norm": 0.04999293751073471, "target_actual_norm": 9.739886283874512, "match_error": 7.152557373046875e-05, "scalar": 9.739886283874512, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1693, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.828255954394685, -0.004713435061826444], "logp": -4.832969389456511, "cache_id": 1694, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.739886283874512, "actual_norm": 9.739957809448242, "relative_norm": 0.04999293751073471, "target_actual_norm": 9.739886283874512, "match_error": 7.152557373046875e-05, "scalar": 9.739886283874512, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.953255954394685, -8.846940030283822e-09], "logp": -4.953255963241625, "cache_id": 1695, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.739886283874512, "actual_norm": 9.739957809448242, "relative_norm": 0.04999293751073471, "target_actual_norm": 9.739886283874512, "match_error": 7.152557373046875e-05, "scalar": 9.739886283874512, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.12028657378511376, "first_log_odds": -0.125, "later_log_odds": 0.0047134262148862405, "first_differing_log_odds": -0.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-unrelated.npz", "array_sha256": "94b9226b773daac7680e7ee9af7c1b7a03922bb034cdd4116f70a86343bbf474", "seconds": 0.31643995782360435, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457796608, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-ordinary", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.734535217285156, "actual_norm": 9.739890098571777, "relative_norm": 0.04999258996655822, "target_actual_norm": 9.739886283874512, "match_error": 3.814697265625e-06, "scalar": 9.73453600454377, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1696, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.913827611928966, -0.0033163538303026116], "logp": -4.917143965759268, "cache_id": 1697, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.734535217285156, "actual_norm": 9.739890098571777, "relative_norm": 0.04999258996655822, "target_actual_norm": 9.739886283874512, "match_error": 3.814697265625e-06, "scalar": 9.73453600454377, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.413827611928966, -1.0110088270972549e-08], "logp": -4.4138276220390535, "cache_id": 1698, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.734535217285156, "actual_norm": 9.739890098571777, "relative_norm": 0.04999258996655822, "target_actual_norm": 9.739886283874512, "match_error": 3.814697265625e-06, "scalar": 9.73453600454377, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.5033163437202148, "first_log_odds": 0.5, "later_log_odds": 0.0033163437202148316, "first_differing_log_odds": 0.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-ordinary.npz", "array_sha256": "1775289de8f7e971859793312fd40c25929bcef111513b35e3e4f9af05f10109", "seconds": 0.31739754136651754, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457829376, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-random0", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.735556602478027, "actual_norm": 9.740015983581543, "relative_norm": 0.049993236105027185, "target_actual_norm": 9.739886283874512, "match_error": 0.00012969970703125, "scalar": 9.735557759277071, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1699, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-4.9970959274760745, -0.0037041790714592575], "logp": -5.000800106547533, "cache_id": 1700, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.735556602478027, "actual_norm": 9.740015983581543, "relative_norm": 0.049993236105027185, "target_actual_norm": 9.739886283874512, "match_error": 0.00012969970703125, "scalar": 9.735557759277071, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.7470959274760745, -9.518073395220722e-09], "logp": -4.747095936994148, "cache_id": 1701, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.735556602478027, "actual_norm": 9.740015983581543, "relative_norm": 0.049993236105027185, "target_actual_norm": 9.739886283874512, "match_error": 0.00012969970703125, "scalar": 9.735557759277071, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.2537041695533855, "first_log_odds": 0.25, "later_log_odds": 0.0037041695533854835, "first_differing_log_odds": 0.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-random0.npz", "array_sha256": "1fa93240f95a72e3db83c08de62db6e5f0d90f71575ded08a05c5f4a16cc57ef", "seconds": 0.31477637495845556, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457829376, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-random1", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.740924835205078, "actual_norm": 9.7400541305542, "relative_norm": 0.049993431904563235, "target_actual_norm": 9.739886283874512, "match_error": 0.0001678466796875, "scalar": 9.7409266159666, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1702, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-5.157519631874796, -0.004198547040941406], "logp": -5.161718178915737, "cache_id": 1703, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.740924835205078, "actual_norm": 9.7400541305542, "relative_norm": 0.049993431904563235, "target_actual_norm": 9.739886283874512, "match_error": 0.0001678466796875, "scalar": 9.7409266159666, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.907519631874796, -1.0796909974408817e-08], "logp": -4.907519642671706, "cache_id": 1704, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.740924835205078, "actual_norm": 9.7400541305542, "relative_norm": 0.049993431904563235, "target_actual_norm": 9.739886283874512, "match_error": 0.0001678466796875, "scalar": 9.7409266159666, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.2541985362440311, "first_log_odds": 0.25, "later_log_odds": 0.004198536244031104, "first_differing_log_odds": 0.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-random1.npz", "array_sha256": "855e1c53bd7fb5e970b5ff61d333abe4241cc91a50b86206ff3f9b8b14ef8fa2", "seconds": 0.31444933265447617, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457829376, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-random2", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "USD<|im_end|>", "output_text": "USD", "output_ids": [26749, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.731340408325195, "actual_norm": 9.739921569824219, "relative_norm": 0.049992751501175464, "target_actual_norm": 9.739886283874512, "match_error": 3.528594970703125e-05, "scalar": 9.731340698832355, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}], "cache_id": 1705, "cached": true, "scoring": {"normalized": "USD", "bare_format": true, "malformed": false, "first_code": "USD", "all_codes": ["USD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-5.116209200409282, -0.00448851553858957], "logp": -5.1206977159478715, "cache_id": 1706, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.731340408325195, "actual_norm": 9.739921569824219, "relative_norm": 0.049992751501175464, "target_actual_norm": 9.739886283874512, "match_error": 3.528594970703125e-05, "scalar": 9.731340698832355, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-5.366209200409282, -9.386223532107107e-09], "logp": -5.366209209795505, "cache_id": 1707, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 194.82667541503906, "strength_one_norm": 35.806373596191406, "target_shrink_scale": 0.2720558602390303, "requested_norm": 9.731340408325195, "actual_norm": 9.739921569824219, "relative_norm": 0.049992751501175464, "target_actual_norm": 9.739886283874512, "match_error": 3.528594970703125e-05, "scalar": 9.731340698832355, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.2455114938476335, "first_log_odds": -0.25, "later_log_odds": 0.0044885061523665115, "first_differing_log_odds": -0.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-random2.npz", "array_sha256": "bcbe1bd7fff320982f416f84cfa85e454b7dc70743ba0cd24b660e35d3729035", "seconds": 0.3182016252540052, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457829376, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-main-bridge", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 406.0985412597656, 307.68511962890625, 13.11544132232666, 142.54551696777344, 80.77665710449219, 77.99618530273438, 14.704142570495605, 47.78351593017578, 234.09764099121094, 181.39938354492188, 131.53948974609375, 54.1275520324707, 103.96791076660156, 58.763545989990234, 64.00186157226562, 96.43184661865234, 246.28314208984375, 89.33052062988281, 498.4502868652344, 360.6343994140625, 425.1349792480469, 13.258085250854492, 385.173095703125, 319.55181884765625, 355.5713806152344, 465.8818664550781, 293.4156799316406]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 406.0943908691406, 307.7286682128906, 13.168597221374512, 142.54342651367188, 80.77884674072266, 78.00267028808594, 14.766359329223633, 47.75747299194336, 234.05194091796875, 181.36119079589844, 131.5159149169922, 54.086708068847656, 103.99177551269531, 58.78200912475586, 64.0131607055664, 96.43522644042969, 246.33082580566406, 89.32181549072266, 498.4152526855469, 360.6827392578125, 425.0828552246094, 13.35252857208252, 385.1632385253906, 319.5191345214844, 355.6120300292969, 465.9148864746094, 293.4111633300781]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.38497209548950195, 0.3092902600765228, 0.013862915337085724, 0.14175060391426086, 0.07581290602684021, 0.07633141428232193, 0.014308270998299122, 0.04210791364312172, 0.23053979873657227, 0.1739378273487091, 0.13395017385482788, 0.056064292788505554, 0.10484343022108078, 0.056877341121435165, 0.061129357665777206, 0.07837679982185364, 0.24950388073921204, 0.1004522517323494, 0.5512107610702515, 0.4303915798664093, 0.4222354292869568, 0.014235883951187134, 0.3415915369987488, 0.3079858720302582, 0.3806012272834778, 0.4497108459472656, 0.2758276164531708]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1708, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-18.019266191954088, -0.0023696783157829693], "logp": -18.021635870269872, "cache_id": 1709, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 406.0985412597656, 307.68511962890625, 13.11544132232666, 142.54551696777344, 80.77665710449219, 77.99618530273438, 14.704142570495605, 47.78351593017578, 234.09764099121094, 181.39938354492188, 131.53948974609375, 54.1275520324707, 103.96791076660156, 58.763545989990234, 64.00186157226562, 96.43184661865234, 246.28314208984375, 89.33052062988281, 498.4502868652344, 360.6343994140625, 425.1349792480469, 13.258085250854492, 385.173095703125, 319.55181884765625, 355.5713806152344, 465.8818664550781, 293.4156799316406]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 406.0943908691406, 307.7286682128906, 13.168597221374512, 142.54342651367188, 80.77884674072266, 78.00267028808594, 14.766359329223633, 47.75747299194336, 234.05194091796875, 181.36119079589844, 131.5159149169922, 54.086708068847656, 103.99177551269531, 58.78200912475586, 64.0131607055664, 96.43522644042969, 246.33082580566406, 89.32181549072266, 498.4152526855469, 360.6827392578125, 425.0828552246094, 13.35252857208252, 385.1632385253906, 319.5191345214844, 355.6120300292969, 465.9148864746094, 293.4111633300781]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.38497209548950195, 0.3092902600765228, 0.013862915337085724, 0.14175060391426086, 0.07581290602684021, 0.07633141428232193, 0.014308270998299122, 0.04210791364312172, 0.23053979873657227, 0.1739378273487091, 0.13395017385482788, 0.056064292788505554, 0.10484343022108078, 0.056877341121435165, 0.061129357665777206, 0.07837679982185364, 0.24950388073921204, 0.1004522517323494, 0.5512107610702515, 0.4303915798664093, 0.4222354292869568, 0.014235883951187134, 0.3415915369987488, 0.3079858720302582, 0.3806012272834778, 0.4497108459472656, 0.2758276164531708]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.019266191954087245, -1.6458245519778442e-08], "logp": -0.019266208412332765, "cache_id": 1710, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 406.0985412597656, 307.68511962890625, 13.11544132232666, 142.54551696777344, 80.77665710449219, 77.99618530273438, 14.704142570495605, 47.78351593017578, 234.09764099121094, 181.39938354492188, 131.53948974609375, 54.1275520324707, 103.96791076660156, 58.763545989990234, 64.00186157226562, 96.43184661865234, 246.28314208984375, 89.33052062988281, 498.4502868652344, 360.6343994140625, 425.1349792480469, 13.258085250854492, 385.173095703125, 319.55181884765625, 355.5713806152344, 465.8818664550781, 293.4156799316406]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 406.0943908691406, 307.7286682128906, 13.168597221374512, 142.54342651367188, 80.77884674072266, 78.00267028808594, 14.766359329223633, 47.75747299194336, 234.05194091796875, 181.36119079589844, 131.5159149169922, 54.086708068847656, 103.99177551269531, 58.78200912475586, 64.0131607055664, 96.43522644042969, 246.33082580566406, 89.32181549072266, 498.4152526855469, 360.6827392578125, 425.0828552246094, 13.35252857208252, 385.1632385253906, 319.5191345214844, 355.6120300292969, 465.9148864746094, 293.4111633300781]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.38497209548950195, 0.3092902600765228, 0.013862915337085724, 0.14175060391426086, 0.07581290602684021, 0.07633141428232193, 0.014308270998299122, 0.04210791364312172, 0.23053979873657227, 0.1739378273487091, 0.13395017385482788, 0.056064292788505554, 0.10484343022108078, 0.056877341121435165, 0.061129357665777206, 0.07837679982185364, 0.24950388073921204, 0.1004522517323494, 0.5512107610702515, 0.4303915798664093, 0.4222354292869568, 0.014235883951187134, 0.3415915369987488, 0.3079858720302582, 0.3806012272834778, 0.4497108459472656, 0.2758276164531708]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 18.00236966185754, "first_log_odds": 18.0, "later_log_odds": 0.0023696618575392847, "first_differing_log_odds": 18.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-main-bridge.npz", "array_sha256": "5606efff1b931928b4ba923d2835199aa4fd153652a67b808c5b85a936833135", "seconds": 0.3544239164330065, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-baseline", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1711, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.62500681199257, -0.0001341255130526959], "logp": -20.625140937505623, "cache_id": 1712, "trace": []}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.12500681199257, -1.5232002875789276e-07], "logp": -22.1250069643126, "cache_id": 1713, "trace": []}, "log_odds": -1.499866026806977, "first_log_odds": -1.5, "later_log_odds": 0.00013397319302299593, "first_differing_log_odds": -1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-baseline.npz", "array_sha256": "39853b442d4d3d7c3ef4283804899b2e576d7391b86ff5edeefe2d5259c83ff1", "seconds": 1.2035863753408194, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-noop", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.708834648132324, "match_error": -9.708834648132324, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1714, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.62500681199257, -0.0001341255130526959], "logp": -20.625140937505623, "cache_id": 1715, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.708834648132324, "match_error": -9.708834648132324, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.12500681199257, -1.5232002875789276e-07], "logp": -22.1250069643126, "cache_id": 1716, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.708834648132324, "match_error": -9.708834648132324, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.499866026806977, "first_log_odds": -1.5, "later_log_odds": 0.00013397319302299593, "first_differing_log_odds": -1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-noop.npz", "array_sha256": "c6ba54a91d84dd3ba552fc5d16041f7a6147fad6a23957594ed3efb90c8c3830", "seconds": 1.2287456248886883, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-j", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.71882152557373, "actual_norm": 9.708834648132324, "relative_norm": 0.049948619964087254, "target_actual_norm": 9.708834648132324, "match_error": 0.0, "scalar": 0.2457975134201968, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1717, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.8750086743157, -0.00011002035334744528], "logp": -20.875118694669048, "cache_id": 1718, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.71882152557373, "actual_norm": 9.708834648132324, "relative_norm": 0.049948619964087254, "target_actual_norm": 9.708834648132324, "match_error": 0.0, "scalar": 0.2457975134201968, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-20.0000086743157, -1.081868003281072e-07], "logp": -20.0000087825025, "cache_id": 1719, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.71882152557373, "actual_norm": 9.708834648132324, "relative_norm": 0.049948619964087254, "target_actual_norm": 9.708834648132324, "match_error": 0.0, "scalar": 0.2457975134201968, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 0.8751099121665469, "first_log_odds": 0.875, "later_log_odds": 0.00010991216654687719, "first_differing_log_odds": 0.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-j.npz", "array_sha256": "9ca057d4af335e9fb983d9156c769ff74cfbf853e3a226493c08a7120ddca4ee", "seconds": 1.2250406248494983, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-opposite", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704316139221191, "actual_norm": 9.708708763122559, "relative_norm": 0.04994797232894464, "target_actual_norm": 9.708834648132324, "match_error": -0.000125885009765625, "scalar": 9.704316223804199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1720, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.000008278850643, -0.00010860734620607493], "logp": -20.00011688619685, "cache_id": 1721, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704316139221191, "actual_norm": 9.708708763122559, "relative_norm": 0.04994797232894464, "target_actual_norm": 9.708834648132324, "match_error": -0.000125885009765625, "scalar": 9.704316223804199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-23.625008278850643, -1.5779558747018328e-07], "logp": -23.62500843664623, "cache_id": 1722, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704316139221191, "actual_norm": 9.708708763122559, "relative_norm": 0.04994797232894464, "target_actual_norm": 9.708834648132324, "match_error": -0.000125885009765625, "scalar": 9.704316223804199, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -3.62489155044938, "first_log_odds": -3.625, "later_log_odds": 0.00010844955061983796, "first_differing_log_odds": -3.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-opposite.npz", "array_sha256": "83b455f53e3ce9f690af899b4789a2df824941e1628b64d3c7e4a54b67020d8c", "seconds": 1.2174785840325058, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-unrelated", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704833030700684, "actual_norm": 9.708601951599121, "relative_norm": 0.04994742282033879, "target_actual_norm": 9.708834648132324, "match_error": -0.000232696533203125, "scalar": 9.704834731513984, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1723, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.375006599462573, -0.00013326768710535832], "logp": -20.375139867149677, "cache_id": 1724, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704833030700684, "actual_norm": 9.708601951599121, "relative_norm": 0.04994742282033879, "target_actual_norm": 9.708834648132324, "match_error": -0.000232696533203125, "scalar": 9.704834731513984, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.000006599462573, -1.4404216692283684e-07], "logp": -22.00000674350474, "cache_id": 1725, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.704833030700684, "actual_norm": 9.708601951599121, "relative_norm": 0.04994742282033879, "target_actual_norm": 9.708834648132324, "match_error": -0.000232696533203125, "scalar": 9.704834731513984, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.624866876355064, "first_log_odds": -1.625, "later_log_odds": 0.00013312364493600626, "first_differing_log_odds": -1.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-unrelated.npz", "array_sha256": "3bddf10ee78d113628adff540fc0695b0756f23312ffefd620fa1bc6d5efe03d", "seconds": 1.2299353331327438, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-ordinary", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.700517654418945, "actual_norm": 9.708868026733398, "relative_norm": 0.049948791685526586, "target_actual_norm": 9.708834648132324, "match_error": 3.337860107421875e-05, "scalar": 9.700520006643274, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1726, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.625007004493607, -0.000153298555619468], "logp": -20.625160303049228, "cache_id": 1727, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.700517654418945, "actual_norm": 9.708868026733398, "relative_norm": 0.049948791685526586, "target_actual_norm": 9.708834648132324, "match_error": 3.337860107421875e-05, "scalar": 9.700520006643274, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-21.625007004493607, -1.736021025013804e-07], "logp": -21.62500717809571, "cache_id": 1728, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.700517654418945, "actual_norm": 9.708868026733398, "relative_norm": 0.049948791685526586, "target_actual_norm": 9.708834648132324, "match_error": 3.337860107421875e-05, "scalar": 9.700520006643274, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.9998468750464831, "first_log_odds": -1.0, "later_log_odds": 0.00015312495351693656, "first_differing_log_odds": -1.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-ordinary.npz", "array_sha256": "25ff394b3adc4f3df84530458248abd7144c3b53e8be0e9319cc0f4aedd58bfc", "seconds": 1.2503403327427804, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-random0", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.713499069213867, "actual_norm": 9.7087984085083, "relative_norm": 0.049948433523667415, "target_actual_norm": 9.708834648132324, "match_error": -3.62396240234375e-05, "scalar": 9.713501217520388, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1729, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.37500793616916, -0.00013267142145948589], "logp": -20.375140607590616, "cache_id": 1730, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.713499069213867, "actual_norm": 9.7087984085083, "relative_norm": 0.049948433523667415, "target_actual_norm": 9.708834648132324, "match_error": -3.62396240234375e-05, "scalar": 9.713501217520388, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-21.75000793616916, -1.367627628661637e-07], "logp": -21.75000807293192, "cache_id": 1731, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.713499069213867, "actual_norm": 9.7087984085083, "relative_norm": 0.049948433523667415, "target_actual_norm": 9.708834648132324, "match_error": -3.62396240234375e-05, "scalar": 9.713501217520388, "random_seed": 20273808, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.3748674653413033, "first_log_odds": -1.375, "later_log_odds": 0.00013253465869667025, "first_differing_log_odds": -1.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-random0.npz", "array_sha256": "00cb3dfd13150b88beb3a876d74395d17ca1a8ca5f84dd6803f4fac72a23f478", "seconds": 1.229594917036593, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-random1", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70727252960205, "actual_norm": 9.709094047546387, "relative_norm": 0.04994995448498719, "target_actual_norm": 9.708834648132324, "match_error": 0.0002593994140625, "scalar": 9.707272180703285, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1732, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.62500578195085, -0.00010284793079575014], "logp": -20.625108629881645, "cache_id": 1733, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70727252960205, "actual_norm": 9.709094047546387, "relative_norm": 0.04994995448498719, "target_actual_norm": 9.708834648132324, "match_error": 0.0002593994140625, "scalar": 9.707272180703285, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.12500578195085, -1.4544834611444604e-07], "logp": -22.125005927399197, "cache_id": 1734, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70727252960205, "actual_norm": 9.709094047546387, "relative_norm": 0.04994995448498719, "target_actual_norm": 9.708834648132324, "match_error": 0.0002593994140625, "scalar": 9.707272180703285, "random_seed": 20273809, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.4998972975175526, "first_log_odds": -1.5, "later_log_odds": 0.0001027024824473699, "first_differing_log_odds": -1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-random1.npz", "array_sha256": "a36d0fe2133dcf3c6931aa77d1fc12086b8bc8f0686861c4623f47488a405921", "seconds": 1.225151624996215, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-random2", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70053768157959, "actual_norm": 9.708573341369629, "relative_norm": 0.04994727563053365, "target_actual_norm": 9.708834648132324, "match_error": -0.0002613067626953125, "scalar": 9.700538524775766, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1735, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-20.750004719197076, -0.00013454632993463732], "logp": -20.75013926552701, "cache_id": 1736, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70053768157959, "actual_norm": 9.708573341369629, "relative_norm": 0.04994727563053365, "target_actual_norm": 9.708834648132324, "match_error": -0.0002613067626953125, "scalar": 9.700538524775766, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.625004719197076, -1.273221755795839e-07], "logp": -22.62500484651925, "cache_id": 1737, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 194.37643432617188, "strength_one_norm": 39.53995132446289, "target_shrink_scale": 0.2457975134201968, "requested_norm": 9.70053768157959, "actual_norm": 9.708573341369629, "relative_norm": 0.04994727563053365, "target_actual_norm": 9.708834648132324, "match_error": -0.0002613067626953125, "scalar": 9.700538524775766, "random_seed": 20273810, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.874865580992239, "first_log_odds": -1.875, "later_log_odds": 0.0001344190077610108, "first_differing_log_odds": -1.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-random2.npz", "array_sha256": "a6ff4a2eea58ca8ea6f06ac1a584a3b663712df0f5efe7d72190b4ca37b1f5da", "seconds": 1.2321250420063734, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-29-paraphrase-bridge", "case_id": "eval-29", "city": "Kuala Lumpur", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 86333, 86244, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 442.60302734375, 332.2193908691406, 22.99830436706543, 107.62171173095703, 51.869964599609375, 108.85954284667969, 31.557619094848633, 53.165443420410156, 244.44419860839844, 154.52906799316406, 139.73236083984375, 62.168540954589844, 132.15162658691406, 12.005648612976074, 44.57918930053711, 475.9872131347656, 381.2106018066406, 422.5614929199219, 69.6436767578125, 377.76702880859375, 358.6944580078125, 340.321044921875, 471.98583984375, 141.41673278808594]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 442.66412353515625, 332.205078125, 22.987627029418945, 107.60552215576172, 51.91482925415039, 108.88460540771484, 31.594511032104492, 53.21204376220703, 244.5089874267578, 154.52206420898438, 139.76470947265625, 62.21229553222656, 132.2146453857422, 12.08604621887207, 44.62057876586914, 475.97772216796875, 381.2203369140625, 422.5707092285156, 69.72623443603516, 377.7008972167969, 358.7543640136719, 340.35003662109375, 471.9882507324219, 141.51039123535156]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.42056772112846375, 0.3316957652568817, 0.02292896807193756, 0.09890209883451462, 0.04933236539363861, 0.1005244255065918, 0.03059343807399273, 0.04597296938300133, 0.2319784164428711, 0.14590905606746674, 0.1414075642824173, 0.06406039744615555, 0.1304388791322708, 0.011460483074188232, 0.046370554715394974, 0.5354827046394348, 0.4387994706630707, 0.41316136717796326, 0.07420847564935684, 0.33170148730278015, 0.33938151597976685, 0.35985544323921204, 0.4562113583087921, 0.14589941501617432]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1738, "cached": true, "scoring": {"normalized": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Kuala Lumpur is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 41, "end": 50}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-28.375001493035956, -0.0001748237800623289], "logp": -28.37517631681602, "cache_id": 1739, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 442.60302734375, 332.2193908691406, 22.99830436706543, 107.62171173095703, 51.869964599609375, 108.85954284667969, 31.557619094848633, 53.165443420410156, 244.44419860839844, 154.52906799316406, 139.73236083984375, 62.168540954589844, 132.15162658691406, 12.005648612976074, 44.57918930053711, 475.9872131347656, 381.2106018066406, 422.5614929199219, 69.6436767578125, 377.76702880859375, 358.6944580078125, 340.321044921875, 471.98583984375, 141.41673278808594]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 442.66412353515625, 332.205078125, 22.987627029418945, 107.60552215576172, 51.91482925415039, 108.88460540771484, 31.594511032104492, 53.21204376220703, 244.5089874267578, 154.52206420898438, 139.76470947265625, 62.21229553222656, 132.2146453857422, 12.08604621887207, 44.62057876586914, 475.97772216796875, 381.2203369140625, 422.5707092285156, 69.72623443603516, 377.7008972167969, 358.7543640136719, 340.35003662109375, 471.9882507324219, 141.51039123535156]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.42056772112846375, 0.3316957652568817, 0.02292896807193756, 0.09890209883451462, 0.04933236539363861, 0.1005244255065918, 0.03059343807399273, 0.04597296938300133, 0.2319784164428711, 0.14590905606746674, 0.1414075642824173, 0.06406039744615555, 0.1304388791322708, 0.011460483074188232, 0.046370554715394974, 0.5354827046394348, 0.4387994706630707, 0.41316136717796326, 0.07420847564935684, 0.33170148730278015, 0.33938151597976685, 0.35985544323921204, 0.4562113583087921, 0.14589941501617432]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-18.125001493035956, -2.1020004459389735e-07], "logp": -18.125001703236002, "cache_id": 1740, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 442.60302734375, 332.2193908691406, 22.99830436706543, 107.62171173095703, 51.869964599609375, 108.85954284667969, 31.557619094848633, 53.165443420410156, 244.44419860839844, 154.52906799316406, 139.73236083984375, 62.168540954589844, 132.15162658691406, 12.005648612976074, 44.57918930053711, 475.9872131347656, 381.2106018066406, 422.5614929199219, 69.6436767578125, 377.76702880859375, 358.6944580078125, 340.321044921875, 471.98583984375, 141.41673278808594]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 442.66412353515625, 332.205078125, 22.987627029418945, 107.60552215576172, 51.91482925415039, 108.88460540771484, 31.594511032104492, 53.21204376220703, 244.5089874267578, 154.52206420898438, 139.76470947265625, 62.21229553222656, 132.2146453857422, 12.08604621887207, 44.62057876586914, 475.97772216796875, 381.2203369140625, 422.5707092285156, 69.72623443603516, 377.7008972167969, 358.7543640136719, 340.35003662109375, 471.9882507324219, 141.51039123535156]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.42056772112846375, 0.3316957652568817, 0.02292896807193756, 0.09890209883451462, 0.04933236539363861, 0.1005244255065918, 0.03059343807399273, 0.04597296938300133, 0.2319784164428711, 0.14590905606746674, 0.1414075642824173, 0.06406039744615555, 0.1304388791322708, 0.011460483074188232, 0.046370554715394974, 0.5354827046394348, 0.4387994706630707, 0.41316136717796326, 0.07420847564935684, 0.33170148730278015, 0.33938151597976685, 0.35985544323921204, 0.4562113583087921, 0.14589941501617432]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 10.250174613580018, "first_log_odds": 10.25, "later_log_odds": 0.00017461358001824578, "first_differing_log_odds": 10.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-29-paraphrase-bridge.npz", "array_sha256": "69892fca3b35078fa4c636244373d394fef31c3b6e5bf6ff149ed7cf879237ea", "seconds": 1.2313638753257692, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-baseline", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1741, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-1.9733347928537275, -0.011133985368576354], "logp": -1.9844687782223038, "cache_id": 1742, "trace": []}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.0983347928537275, -8.376399733843854e-08], "logp": -3.098334876617725, "cache_id": 1743, "trace": []}, "log_odds": -1.1138660983954212, "first_log_odds": -1.125, "later_log_odds": 0.011133901604578833, "first_differing_log_odds": -1.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-baseline.npz", "array_sha256": "ef6779cb582c34f312c1eca533d920e1fe0e4cb45715107ee5b70c75f5e5b1e2", "seconds": 0.389464917127043, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-noop", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.998345375061035, "match_error": -9.998345375061035, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1744, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-1.9733347928537275, -0.011133985368576354], "logp": -1.9844687782223038, "cache_id": 1745, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.998345375061035, "match_error": -9.998345375061035, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.0983347928537275, -8.376399733843854e-08], "logp": -3.098334876617725, "cache_id": 1746, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.998345375061035, "match_error": -9.998345375061035, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.1138660983954212, "first_log_odds": -1.125, "later_log_odds": 0.011133901604578833, "first_differing_log_odds": -1.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-noop.npz", "array_sha256": "f46191204ac27715514bc876479e106417e3612b9e44c9a3e34cd48e3013e16a", "seconds": 0.33784441696479917, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-j", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.000862121582031, "actual_norm": 9.998345375061035, "relative_norm": 0.049987424978990454, "target_actual_norm": 9.998345375061035, "match_error": 0.0, "scalar": 0.21241305105207295, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1747, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-3.0869904083155837, -0.009621385747986898], "logp": -3.0966117940635707, "cache_id": 1748, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.000862121582031, "actual_norm": 9.998345375061035, "relative_norm": 0.049987424978990454, "target_actual_norm": 9.998345375061035, "match_error": 0.0, "scalar": 0.21241305105207295, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.5869904083155838, -6.40458296981924e-08], "logp": -0.5869904723614136, "cache_id": 1749, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.000862121582031, "actual_norm": 9.998345375061035, "relative_norm": 0.049987424978990454, "target_actual_norm": 9.998345375061035, "match_error": 0.0, "scalar": 0.21241305105207295, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 2.509621321702157, "first_log_odds": 2.5, "later_log_odds": 0.009621321702157015, "first_differing_log_odds": 2.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-j.npz", "array_sha256": "c92a7f4c16a40656387a4442dfd787e184d9ad8de7b34fcebcee14a2c6611f3a", "seconds": 0.34008600004017353, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-opposite", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.99478816986084, "actual_norm": 9.999170303344727, "relative_norm": 0.04999154926547459, "target_actual_norm": 9.998345375061035, "match_error": 0.0008249282836914062, "scalar": 9.994788758444884, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1750, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-2.2837999882690347, -0.008156019640717816], "logp": -2.2919560079097527, "cache_id": 1751, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.99478816986084, "actual_norm": 9.999170303344727, "relative_norm": 0.04999154926547459, "target_actual_norm": 9.998345375061035, "match_error": 0.0008249282836914062, "scalar": 9.994788758444884, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-6.283799988269035, -1.1778232428516715e-07], "logp": -6.283800106051359, "cache_id": 1752, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.99478816986084, "actual_norm": 9.999170303344727, "relative_norm": 0.04999154926547459, "target_actual_norm": 9.998345375061035, "match_error": 0.0008249282836914062, "scalar": 9.994788758444884, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -3.9918440981416063, "first_log_odds": -4.0, "later_log_odds": 0.008155901858393655, "first_differing_log_odds": -4.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-opposite.npz", "array_sha256": "71985ae64ab86dcf78bb47e45153f662b1b272b196aba7fad01c6a9fc47a431a", "seconds": 0.34056799998506904, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-unrelated", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.005743026733398, "actual_norm": 9.998320579528809, "relative_norm": 0.049987301011997866, "target_actual_norm": 9.998345375061035, "match_error": -2.47955322265625e-05, "scalar": 10.005744663249061, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1753, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-1.813599035223602, -0.01132015394458127], "logp": -1.8249191891681833, "cache_id": 1754, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.005743026733398, "actual_norm": 9.998320579528809, "relative_norm": 0.049987301011997866, "target_actual_norm": 9.998345375061035, "match_error": -2.47955322265625e-05, "scalar": 10.005744663249061, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-2.9385990352236018, -1.1431653614232705e-07], "logp": -2.938599149540138, "cache_id": 1755, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 10.005743026733398, "actual_norm": 9.998320579528809, "relative_norm": 0.049987301011997866, "target_actual_norm": 9.998345375061035, "match_error": -2.47955322265625e-05, "scalar": 10.005744663249061, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.1136799603719547, "first_log_odds": -1.1249999999999998, "later_log_odds": 0.011320039628045064, "first_differing_log_odds": -1.1249999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-unrelated.npz", "array_sha256": "6599553ece386adaadbec44300860d648b7c59e46a7d21f942e84ae2ec97f372", "seconds": 0.3403850416652858, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-ordinary", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.992185592651367, "actual_norm": 9.99829387664795, "relative_norm": 0.049987167509082775, "target_actual_norm": 9.998345375061035, "match_error": -5.14984130859375e-05, "scalar": 9.992185658347807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1756, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-1.983001227919638, -0.010635905930486028], "logp": -1.9936371338501242, "cache_id": 1757, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.992185592651367, "actual_norm": 9.99829387664795, "relative_norm": 0.049987167509082775, "target_actual_norm": 9.998345375061035, "match_error": -5.14984130859375e-05, "scalar": 9.992185658347807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-2.233001227919638, -9.492862955885026e-08], "logp": -2.2330013228482675, "cache_id": 1758, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.992185592651367, "actual_norm": 9.99829387664795, "relative_norm": 0.049987167509082775, "target_actual_norm": 9.998345375061035, "match_error": -5.14984130859375e-05, "scalar": 9.992185658347807, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.23936418899814327, "first_log_odds": -0.25, "later_log_odds": 0.010635811001856732, "first_differing_log_odds": -0.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-ordinary.npz", "array_sha256": "1cc61659eca91e53e4ceb11aa7a03bd099a646c696fc09a8620762fb290816d2", "seconds": 0.33665050007402897, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-random0", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.994149208068848, "actual_norm": 9.998339653015137, "relative_norm": 0.04998739637122294, "target_actual_norm": 9.998345375061035, "match_error": -5.7220458984375e-06, "scalar": 9.994149902377103, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1759, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-1.8853465522299482, -0.010340813426069089], "logp": -1.8956873656560174, "cache_id": 1760, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.994149208068848, "actual_norm": 9.998339653015137, "relative_norm": 0.04998739637122294, "target_actual_norm": 9.998345375061035, "match_error": -5.7220458984375e-06, "scalar": 9.994149902377103, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.260346552229948, -1.035330829652311e-07], "logp": -3.260346655763031, "cache_id": 1761, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.994149208068848, "actual_norm": 9.998339653015137, "relative_norm": 0.04998739637122294, "target_actual_norm": 9.998345375061035, "match_error": -5.7220458984375e-06, "scalar": 9.994149902377103, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.3646592901070138, "first_log_odds": -1.375, "later_log_odds": 0.010340709892986188, "first_differing_log_odds": -1.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-random0.npz", "array_sha256": "6f63afc23bab3adbb6f67bb4caf3657d4075054d9f8535b43ce0798e876ec621", "seconds": 0.3366663330234587, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-random1", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.983396530151367, "actual_norm": 9.99852466583252, "relative_norm": 0.049988321355706085, "target_actual_norm": 9.998345375061035, "match_error": 0.000179290771484375, "scalar": 9.983394236041931, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1762, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-2.1073745746930026, -0.007923216420248036], "logp": -2.1152977911132504, "cache_id": 1763, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.983396530151367, "actual_norm": 9.99852466583252, "relative_norm": 0.049988321355706085, "target_actual_norm": 9.998345375061035, "match_error": 0.000179290771484375, "scalar": 9.983394236041931, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-2.8573745746930026, -6.952445910437236e-08], "logp": -2.8573746442174617, "cache_id": 1764, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.983396530151367, "actual_norm": 9.99852466583252, "relative_norm": 0.049988321355706085, "target_actual_norm": 9.998345375061035, "match_error": 0.000179290771484375, "scalar": 9.983394236041931, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.7420768531042112, "first_log_odds": -0.75, "later_log_odds": 0.007923146895788769, "first_differing_log_odds": -0.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-random1.npz", "array_sha256": "6408b6a7c9fe5c3b0bfc1dfb314b9c015197ff5855008974316ae56e47d9c905", "seconds": 0.3380061248317361, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-random2", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "SGD<|im_end|>", "output_text": "SGD", "output_ids": [7783, 35, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.999149322509766, "actual_norm": 9.998404502868652, "relative_norm": 0.049987720592588165, "target_actual_norm": 9.998345375061035, "match_error": 5.91278076171875e-05, "scalar": 9.999146328937059, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1765, "cached": true, "scoring": {"normalized": "SGD", "bare_format": true, "malformed": false, "first_code": "SGD", "all_codes": ["SGD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-2.0320258376970424, -0.011744174237502363], "logp": -2.0437700119345448, "cache_id": 1766, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.999149322509766, "actual_norm": 9.998404502868652, "relative_norm": 0.049987720592588165, "target_actual_norm": 9.998345375061035, "match_error": 5.91278076171875e-05, "scalar": 9.999146328937059, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.5320258376970424, -8.679997898983938e-08], "logp": -3.5320259244970216, "cache_id": 1767, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 200.0172119140625, "strength_one_norm": 47.08213806152344, "target_shrink_scale": 0.21241305105207295, "requested_norm": 9.999149322509766, "actual_norm": 9.998404502868652, "relative_norm": 0.049987720592588165, "target_actual_norm": 9.998345375061035, "match_error": 5.91278076171875e-05, "scalar": 9.999146328937059, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -1.4882559125624768, "first_log_odds": -1.5, "later_log_odds": 0.011744087437523199, "first_differing_log_odds": -1.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-random2.npz", "array_sha256": "990d302a9d3afb65d91ae0a6dbffe8fe6654290333b639ab665a3ec00cdd33e1", "seconds": 0.3385556251741946, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-main-bridge", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 71.91850280761719, 726.9204711914062, 23.594593048095703, 129.3909454345703, 91.88203430175781, 80.53492736816406, 14.60265827178955, 59.1066780090332, 229.60047912597656, 160.50674438476562, 138.58731079101562, 63.375953674316406, 97.05550384521484, 72.84565734863281, 61.769065856933594, 98.44102478027344, 246.79403686523438, 111.95820617675781, 482.9763488769531, 359.1835021972656, 412.2693176269531, 18.568960189819336, 391.0281066894531, 316.3746643066406, 331.7009582519531, 468.982177734375, 400.21728515625]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 71.90689849853516, 727.0155029296875, 23.649349212646484, 129.3506317138672, 91.84275817871094, 80.54750061035156, 14.683050155639648, 59.07980728149414, 229.61434936523438, 160.5186309814453, 138.6016387939453, 63.40869140625, 97.05728149414062, 72.8592300415039, 61.7682991027832, 98.4379653930664, 246.85491943359375, 111.97407531738281, 482.9211730957031, 359.2113342285156, 412.1976623535156, 18.59819793701172, 391.011474609375, 316.40740966796875, 331.6335754394531, 468.9534912109375, 400.2147216796875]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.05989968776702881, 0.7022787928581238, 0.02515154704451561, 0.12930963933467865, 0.08759491890668869, 0.07832632213830948, 0.014220681972801685, 0.05185478553175926, 0.22558081150054932, 0.15312336385250092, 0.14135649800300598, 0.06612696498632431, 0.09760590642690659, 0.07061406224966049, 0.05865240469574928, 0.079571433365345, 0.2483549565076828, 0.12575261294841766, 0.536429762840271, 0.4297819435596466, 0.41226926445961, 0.01968422345817089, 0.3458015024662018, 0.30623310804367065, 0.355972558259964, 0.45237603783607483, 0.3859705626964569]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1768, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "target", "correct_code": false, "clean_source": false, "clean_target": true, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-21.876097029863192, -0.009249509491419209], "logp": -21.88534653935461, "cache_id": 1769, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 71.91850280761719, 726.9204711914062, 23.594593048095703, 129.3909454345703, 91.88203430175781, 80.53492736816406, 14.60265827178955, 59.1066780090332, 229.60047912597656, 160.50674438476562, 138.58731079101562, 63.375953674316406, 97.05550384521484, 72.84565734863281, 61.769065856933594, 98.44102478027344, 246.79403686523438, 111.95820617675781, 482.9763488769531, 359.1835021972656, 412.2693176269531, 18.568960189819336, 391.0281066894531, 316.3746643066406, 331.7009582519531, 468.982177734375, 400.21728515625]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 71.90689849853516, 727.0155029296875, 23.649349212646484, 129.3506317138672, 91.84275817871094, 80.54750061035156, 14.683050155639648, 59.07980728149414, 229.61434936523438, 160.5186309814453, 138.6016387939453, 63.40869140625, 97.05728149414062, 72.8592300415039, 61.7682991027832, 98.4379653930664, 246.85491943359375, 111.97407531738281, 482.9211730957031, 359.2113342285156, 412.1976623535156, 18.59819793701172, 391.011474609375, 316.40740966796875, 331.6335754394531, 468.9534912109375, 400.2147216796875]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.05989968776702881, 0.7022787928581238, 0.02515154704451561, 0.12930963933467865, 0.08759491890668869, 0.07832632213830948, 0.014220681972801685, 0.05185478553175926, 0.22558081150054932, 0.15312336385250092, 0.14135649800300598, 0.06612696498632431, 0.09760590642690659, 0.07061406224966049, 0.05865240469574928, 0.079571433365345, 0.2483549565076828, 0.12575261294841766, 0.536429762840271, 0.4297819435596466, 0.41226926445961, 0.01968422345817089, 0.3458015024662018, 0.30623310804367065, 0.355972558259964, 0.45237603783607483, 0.3859705626964569]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0010970298631911373, -1.1566109867113149e-07], "logp": -0.0010971455242898085, "cache_id": 1770, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 76.94825744628906, 123.56751251220703, 69.98775482177734, 36.59262466430664, 111.56291198730469, 71.91850280761719, 726.9204711914062, 23.594593048095703, 129.3909454345703, 91.88203430175781, 80.53492736816406, 14.60265827178955, 59.1066780090332, 229.60047912597656, 160.50674438476562, 138.58731079101562, 63.375953674316406, 97.05550384521484, 72.84565734863281, 61.769065856933594, 98.44102478027344, 246.79403686523438, 111.95820617675781, 482.9763488769531, 359.1835021972656, 412.2693176269531, 18.568960189819336, 391.0281066894531, 316.3746643066406, 331.7009582519531, 468.982177734375, 400.21728515625]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 77.00443267822266, 123.58338165283203, 69.91851806640625, 36.617958068847656, 111.55233764648438, 71.90689849853516, 727.0155029296875, 23.649349212646484, 129.3506317138672, 91.84275817871094, 80.54750061035156, 14.683050155639648, 59.07980728149414, 229.61434936523438, 160.5186309814453, 138.6016387939453, 63.40869140625, 97.05728149414062, 72.8592300415039, 61.7682991027832, 98.4379653930664, 246.85491943359375, 111.97407531738281, 482.9211730957031, 359.2113342285156, 412.1976623535156, 18.59819793701172, 391.011474609375, 316.40740966796875, 331.6335754394531, 468.9534912109375, 400.2147216796875]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.07068736106157303, 0.10842975229024887, 0.05688242241740227, 0.031342051923274994, 0.09830000251531601, 0.05989968776702881, 0.7022787928581238, 0.02515154704451561, 0.12930963933467865, 0.08759491890668869, 0.07832632213830948, 0.014220681972801685, 0.05185478553175926, 0.22558081150054932, 0.15312336385250092, 0.14135649800300598, 0.06612696498632431, 0.09760590642690659, 0.07061406224966049, 0.05865240469574928, 0.079571433365345, 0.2483549565076828, 0.12575261294841766, 0.536429762840271, 0.4297819435596466, 0.41226926445961, 0.01968422345817089, 0.3458015024662018, 0.30623310804367065, 0.355972558259964, 0.45237603783607483, 0.3859705626964569]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 21.88424939383032, "first_log_odds": 21.875, "later_log_odds": 0.009249393830319974, "first_differing_log_odds": 21.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-main-bridge.npz", "array_sha256": "42532c9109c5173e268d8975b2aef7fefe40c38fa10f183b31c0025451c0e63e", "seconds": 0.33445908315479755, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-baseline", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1771, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-23.37500005389627, -0.0005581163320087828], "logp": -23.375558170228278, "cache_id": 1772, "trace": []}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-21.75000005389627, -1.2295949771286749e-05], "logp": -21.750012349846042, "cache_id": 1773, "trace": []}, "log_odds": 1.6255458203822357, "first_log_odds": 1.625, "later_log_odds": 0.0005458203822357177, "first_differing_log_odds": 1.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-baseline.npz", "array_sha256": "2c69111ce5fabdee14e857c478417d3a1caab91a5854b4fa2288326dd82d9c89", "seconds": 1.2288555409759283, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-noop", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.07365894317627, "match_error": -10.07365894317627, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1774, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-23.37500005389627, -0.0005581163320087828], "logp": -23.375558170228278, "cache_id": 1775, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.07365894317627, "match_error": -10.07365894317627, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-21.75000005389627, -1.2295949771286749e-05], "logp": -21.750012349846042, "cache_id": 1776, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 10.07365894317627, "match_error": -10.07365894317627, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.6255458203822357, "first_log_odds": 1.625, "later_log_odds": 0.0005458203822357177, "first_differing_log_odds": 1.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-noop.npz", "array_sha256": "e33aab6b6383c07f0ffe5a2f4144631665b0eff7af182b89ea86435d3a2488ea", "seconds": 1.2323330417275429, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-j", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.081689834594727, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 0.20841849088936848, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1777, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-24.125000052711066, -0.001009294033350089], "logp": -24.126009346744414, "cache_id": 1778, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.081689834594727, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 0.20841849088936848, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-20.250000052711066, -7.40448000020354e-06], "logp": -20.250007457191067, "cache_id": 1779, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.081689834594727, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 0.20841849088936848, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 3.876001889553347, "first_log_odds": 3.875, "later_log_odds": 0.001001889553347013, "first_differing_log_odds": 3.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-j.npz", "array_sha256": "88634ba080c90008a40a9bf752ceffad886724870ad4f78974eeef7ed7729f89", "seconds": 1.231590375304222, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-opposite", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.08080768585205, "actual_norm": 10.073616981506348, "relative_norm": 0.04995997603045617, "target_actual_norm": 10.07365894317627, "match_error": -4.1961669921875e-05, "scalar": 10.080806543592189, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1780, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-23.75000003894494, -0.00044227350086197817], "logp": -23.750442312445802, "cache_id": 1781, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.08080768585205, "actual_norm": 10.073616981506348, "relative_norm": 0.04995997603045617, "target_actual_norm": 10.07365894317627, "match_error": -4.1961669921875e-05, "scalar": 10.080806543592189, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-23.87500003894494, -3.1842567576123305e-05], "logp": -23.875031881512516, "cache_id": 1782, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.08080768585205, "actual_norm": 10.073616981506348, "relative_norm": 0.04995997603045617, "target_actual_norm": 10.07365894317627, "match_error": -4.1961669921875e-05, "scalar": 10.080806543592189, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -0.12458956906671403, "first_log_odds": -0.125, "later_log_odds": 0.0004104309332859657, "first_differing_log_odds": -0.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-opposite.npz", "array_sha256": "7084bd6b9b22aaa55027f87e1014e22e2d6246e094751a97b194326ca4e7044d", "seconds": 1.222598333377391, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-unrelated", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.072274208068848, "actual_norm": 10.073661804199219, "relative_norm": 0.049960198328034645, "target_actual_norm": 10.07365894317627, "match_error": 2.86102294921875e-06, "scalar": 10.072275536644156, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1783, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-24.250000026650092, -0.0008416860752733786], "logp": -24.250841712725364, "cache_id": 1784, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.072274208068848, "actual_norm": 10.073661804199219, "relative_norm": 0.049960198328034645, "target_actual_norm": 10.07365894317627, "match_error": 2.86102294921875e-06, "scalar": 10.072275536644156, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.625000026650092, -2.0803288017466187e-05], "logp": -22.62502082993811, "cache_id": 1785, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.072274208068848, "actual_norm": 10.073661804199219, "relative_norm": 0.049960198328034645, "target_actual_norm": 10.07365894317627, "match_error": 2.86102294921875e-06, "scalar": 10.072275536644156, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.6258208827872558, "first_log_odds": 1.625, "later_log_odds": 0.0008208827872557833, "first_differing_log_odds": 1.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-unrelated.npz", "array_sha256": "0d434da62f82f29e01c0a44360cbe278ab2eca2238b50eb27d2030d1ad1c7969", "seconds": 1.2523024580441415, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-ordinary", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 11, 27696, 11, 374, 279, 65431, 21525, 12882, 320, 19159, 49, 701, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.062712669372559, "actual_norm": 10.0736722946167, "relative_norm": 0.049960250355127474, "target_actual_norm": 10.07365894317627, "match_error": 1.33514404296875e-05, "scalar": 10.062712979165667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1786, "cached": true, "scoring": {"normalized": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR", "MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysia", "start": 27, "end": 35}, {"text": "Malaysian", "start": 44, "end": 53}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-23.0000000750604, -0.0006138284827527815], "logp": -23.00061390354315, "cache_id": 1787, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.062712669372559, "actual_norm": 10.0736722946167, "relative_norm": 0.049960250355127474, "target_actual_norm": 10.07365894317627, "match_error": 1.33514404296875e-05, "scalar": 10.062712979165667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-21.0000000750604, -9.890965683599017e-06], "logp": -21.00000996602608, "cache_id": 1788, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.062712669372559, "actual_norm": 10.0736722946167, "relative_norm": 0.049960250355127474, "target_actual_norm": 10.07365894317627, "match_error": 1.33514404296875e-05, "scalar": 10.062712979165667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 2.00060393751707, "first_log_odds": 2.0, "later_log_odds": 0.0006039375170701078, "first_differing_log_odds": 2.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-ordinary.npz", "array_sha256": "94a188fef2fa3a843bb7bdd1cfa8c73e9cae9367bd7de5da1c4cf15d435c78dd", "seconds": 1.4114208342507482, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-random0", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.066741943359375, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 10.066741910515702, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1789, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-23.625000047470223, -0.0006731872945929551], "logp": -23.625673234764815, "cache_id": 1790, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.066741943359375, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 10.066741910515702, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.375000047470223, -1.334974121566927e-05], "logp": -22.37501339721144, "cache_id": 1791, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.066741943359375, "actual_norm": 10.07365894317627, "relative_norm": 0.0499601841388275, "target_actual_norm": 10.07365894317627, "match_error": 0.0, "scalar": 10.066741910515702, "random_seed": 20273908, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.2506598375533748, "first_log_odds": 1.25, "later_log_odds": 0.0006598375533748424, "first_differing_log_odds": 1.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-random0.npz", "array_sha256": "57806aed145049f6dfa79c870f7c126160f16db9421761234ef4e580136960a7", "seconds": 1.2317749578505754, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-random1", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.04818344116211, "actual_norm": 10.073358535766602, "relative_norm": 0.049958694272078163, "target_actual_norm": 10.07365894317627, "match_error": -0.00030040740966796875, "scalar": 10.048181206209847, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1792, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-24.1250000403489, -0.0007297625897054664], "logp": -24.125729802938608, "cache_id": 1793, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.04818344116211, "actual_norm": 10.073358535766602, "relative_norm": 0.049958694272078163, "target_actual_norm": 10.07365894317627, "match_error": -0.00030040740966796875, "scalar": 10.048181206209847, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-22.2500000403489, -9.818419254677431e-06], "logp": -22.250009858768156, "cache_id": 1794, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.04818344116211, "actual_norm": 10.073358535766602, "relative_norm": 0.049958694272078163, "target_actual_norm": 10.07365894317627, "match_error": -0.00030040740966796875, "scalar": 10.048181206209847, "random_seed": 20273909, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.8757199441704522, "first_log_odds": 1.875, "later_log_odds": 0.0007199441704521803, "first_differing_log_odds": 1.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-random1.npz", "array_sha256": "db32ac6c4aee2e6f410235be8d386d3738006950d63015c23b1a8f98ceefe7bd", "seconds": 1.2300661662593484, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-random2", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 374, 279, 65431, 21525, 12882, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.070663452148438, "actual_norm": 10.073800086975098, "relative_norm": 0.04996088413971291, "target_actual_norm": 10.07365894317627, "match_error": 0.000141143798828125, "scalar": 10.07066156235669, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1795, "cached": true, "scoring": {"normalized": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh is the Malaysian Ringgit, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysian", "start": 33, "end": 42}], "country_before_code": true, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-24.375000026757462, -0.0006226354437505672], "logp": -24.375622662201213, "cache_id": 1796, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.070663452148438, "actual_norm": 10.073800086975098, "relative_norm": 0.04996088413971291, "target_actual_norm": 10.07365894317627, "match_error": 0.000141143798828125, "scalar": 10.07066156235669, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-23.250000026757462, -1.2469789588790083e-05], "logp": -23.25001249654705, "cache_id": 1797, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 201.6337432861328, "strength_one_norm": 48.3723258972168, "target_shrink_scale": 0.20841849088936848, "requested_norm": 10.070663452148438, "actual_norm": 10.073800086975098, "relative_norm": 0.04996088413971291, "target_actual_norm": 10.07365894317627, "match_error": 0.000141143798828125, "scalar": 10.07066156235669, "random_seed": 20273910, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 1.1256101656541624, "first_log_odds": 1.125, "later_log_odds": 0.0006101656541623868, "first_differing_log_odds": 1.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-random2.npz", "array_sha256": "5749fc537cac2fb793944bf95a1c24a6f6730fbef5c33f71f7fa295d678ee90a", "seconds": 1.2137408331036568, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-30-paraphrase-bridge", "case_id": "eval-30", "city": "Ipoh", "country": "Malaysia", "code": "MYR", "target_country": "Indonesia", "target_code": "IDR", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.<|im_end|>", "output_text": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.", "output_ids": [785, 11413, 1483, 304, 35033, 2267, 11, 27696, 11, 374, 279, 65431, 21525, 12882, 320, 19159, 49, 701, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 19159, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 63.45716857910156, 721.3463745117188, 67.95911407470703, 96.22247314453125, 55.97268295288086, 112.50279235839844, 52.92988967895508, 51.60769271850586, 239.23646545410156, 142.6371307373047, 152.7292022705078, 70.18901062011719, 129.4883270263672, 24.366849899291992, 125.34546661376953, 459.152099609375, 385.7572021484375, 413.34735107421875, 81.6458740234375, 376.7521057128906, 362.0669860839844, 355.2796325683594, 473.2441711425781, 112.47472381591797]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 63.46610641479492, 721.4090576171875, 68.00851440429688, 96.21361541748047, 55.95367431640625, 112.48880767822266, 52.94935607910156, 51.66948318481445, 239.24644470214844, 142.66387939453125, 152.7689971923828, 70.20303344726562, 129.5242919921875, 24.36931610107422, 125.3492431640625, 459.1212158203125, 385.7232360839844, 413.3634948730469, 81.64765167236328, 376.7303466796875, 362.02923583984375, 355.30780029296875, 473.28546142578125, 112.56486511230469]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.052341021597385406, 0.6914130449295044, 0.07301691919565201, 0.09010456502437592, 0.054005127400159836, 0.10321830958127975, 0.050992757081985474, 0.04423794522881508, 0.22511643171310425, 0.13357020914554596, 0.15514735877513885, 0.07236962020397186, 0.12620840966701508, 0.023234523832798004, 0.13566750288009644, 0.5197284817695618, 0.44674596190452576, 0.4055664539337158, 0.08619911968708038, 0.3296169340610504, 0.34399139881134033, 0.3732607066631317, 0.45577099919319153, 0.11733318120241165]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1798, "cached": true, "scoring": {"normalized": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (MYR), and its three-letter ISO 4217 code is **MYR**.", "bare_format": false, "malformed": true, "first_code": "MYR", "all_codes": ["MYR", "MYR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Ipoh, Malaysia, is the Malaysian Ringgit (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Malaysia", "start": 27, "end": 35}, {"text": "Malaysian", "start": 44, "end": 53}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [19159, 49], "tokens": ["MY", "R"], "log_probs": [-29.00000005566091, -0.0006536351130181767], "logp": -29.000653690773927, "cache_id": 1799, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 63.45716857910156, 721.3463745117188, 67.95911407470703, 96.22247314453125, 55.97268295288086, 112.50279235839844, 52.92988967895508, 51.60769271850586, 239.23646545410156, 142.6371307373047, 152.7292022705078, 70.18901062011719, 129.4883270263672, 24.366849899291992, 125.34546661376953, 459.152099609375, 385.7572021484375, 413.34735107421875, 81.6458740234375, 376.7521057128906, 362.0669860839844, 355.2796325683594, 473.2441711425781, 112.47472381591797]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 63.46610641479492, 721.4090576171875, 68.00851440429688, 96.21361541748047, 55.95367431640625, 112.48880767822266, 52.94935607910156, 51.66948318481445, 239.24644470214844, 142.66387939453125, 152.7689971923828, 70.20303344726562, 129.5242919921875, 24.36931610107422, 125.3492431640625, 459.1212158203125, 385.7232360839844, 413.3634948730469, 81.64765167236328, 376.7303466796875, 362.02923583984375, 355.30780029296875, 473.28546142578125, 112.56486511230469]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.052341021597385406, 0.6914130449295044, 0.07301691919565201, 0.09010456502437592, 0.054005127400159836, 0.10321830958127975, 0.050992757081985474, 0.04423794522881508, 0.22511643171310425, 0.13357020914554596, 0.15514735877513885, 0.07236962020397186, 0.12620840966701508, 0.023234523832798004, 0.13566750288009644, 0.5197284817695618, 0.44674596190452576, 0.4055664539337158, 0.08619911968708038, 0.3296169340610504, 0.34399139881134033, 0.3732607066631317, 0.45577099919319153, 0.11733318120241165]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-18.12500005566091, -2.4288093040718937e-05], "logp": -18.12502434375395, "cache_id": 1800, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[163.4325714111328, 121.15208435058594, 72.12345123291016, 14.060861587524414, 73.63131713867188, 135.02020263671875, 36.823177337646484, 98.51360321044922, 63.45716857910156, 721.3463745117188, 67.95911407470703, 96.22247314453125, 55.97268295288086, 112.50279235839844, 52.92988967895508, 51.60769271850586, 239.23646545410156, 142.6371307373047, 152.7292022705078, 70.18901062011719, 129.4883270263672, 24.366849899291992, 125.34546661376953, 459.152099609375, 385.7572021484375, 413.34735107421875, 81.6458740234375, 376.7521057128906, 362.0669860839844, 355.2796325683594, 473.2441711425781, 112.47472381591797]], "actual_norm": [[163.43063354492188, 121.16593933105469, 72.14024353027344, 14.108694076538086, 73.67013549804688, 135.04881286621094, 36.88964080810547, 98.52427673339844, 63.46610641479492, 721.4090576171875, 68.00851440429688, 96.21361541748047, 55.95367431640625, 112.48880767822266, 52.94935607910156, 51.66948318481445, 239.24644470214844, 142.66387939453125, 152.7689971923828, 70.20303344726562, 129.5242919921875, 24.36931610107422, 125.3492431640625, 459.1212158203125, 385.7232360839844, 413.3634948730469, 81.64765167236328, 376.7303466796875, 362.02923583984375, 355.30780029296875, 473.28546142578125, 112.56486511230469]], "relative_norm": [[0.00953860767185688, 0.12853026390075684, 0.07980959862470627, 0.012301306240260601, 0.06706009805202484, 0.11732175946235657, 0.03280922770500183, 0.08751972764730453, 0.052341021597385406, 0.6914130449295044, 0.07301691919565201, 0.09010456502437592, 0.054005127400159836, 0.10321830958127975, 0.050992757081985474, 0.04423794522881508, 0.22511643171310425, 0.13357020914554596, 0.15514735877513885, 0.07236962020397186, 0.12620840966701508, 0.023234523832798004, 0.13566750288009644, 0.5197284817695618, 0.44674596190452576, 0.4055664539337158, 0.08619911968708038, 0.3296169340610504, 0.34399139881134033, 0.3732607066631317, 0.45577099919319153, 0.11733318120241165]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 10.875629347019977, "first_log_odds": 10.875, "later_log_odds": 0.0006293470199771889, "first_differing_log_odds": 10.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-30-paraphrase-bridge.npz", "array_sha256": "4075b8ea76d0f596cdeb0d824def028588306ca36283b0fb1552286976b57c08", "seconds": 1.4263204159215093, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-baseline", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1801, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.00048659672023626104, -5.803326247147421e-09], "logp": -0.00048660252356250817, "cache_id": 1802, "trace": []}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.750486596720236, -5.540476420222856e-05], "logp": -15.750542001484439, "cache_id": 1803, "trace": []}, "log_odds": -15.750055398960876, "first_log_odds": -15.75, "later_log_odds": -5.539896087647378e-05, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-baseline.npz", "array_sha256": "08d14d8a8cf769d4e9f99256a45e60b6a94327bbc4933fdd8e79a362bb6dc8fb", "seconds": 0.37490233266726136, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457862144, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-noop", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.682130813598633, "match_error": -9.682130813598633, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1804, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.00048659672023626104, -5.803326247147421e-09], "logp": -0.00048660252356250817, "cache_id": 1805, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.682130813598633, "match_error": -9.682130813598633, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.750486596720236, -5.540476420222856e-05], "logp": -15.750542001484439, "cache_id": 1806, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.682130813598633, "match_error": -9.682130813598633, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.750055398960876, "first_log_odds": -15.75, "later_log_odds": -5.539896087647378e-05, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-noop.npz", "array_sha256": "9ef51c471a184f061d6700d6a8149fb7b4dbdf4d127a55b2a3886fc5391253df", "seconds": 0.33930729096755385, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457894912, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-j", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.676864624023438, "actual_norm": 9.682130813598633, "relative_norm": 0.04999158768293392, "target_actual_norm": 9.682130813598633, "match_error": 0.0, "scalar": 0.21537794913834993, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1807, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0006947738982695786, -5.786092033191427e-09], "logp": -0.0006947796843616118, "cache_id": 1808, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.676864624023438, "actual_norm": 9.682130813598633, "relative_norm": 0.04999158768293392, "target_actual_norm": 9.682130813598633, "match_error": 0.0, "scalar": 0.21537794913834993, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-14.50069477389827, -2.7012311096542653e-05], "logp": -14.500721786209366, "cache_id": 1809, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.676864624023438, "actual_norm": 9.682130813598633, "relative_norm": 0.04999158768293392, "target_actual_norm": 9.682130813598633, "match_error": 0.0, "scalar": 0.21537794913834993, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -14.500027006525006, "first_log_odds": -14.5, "later_log_odds": -2.7006525005646154e-05, "first_differing_log_odds": -14.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-j.npz", "array_sha256": "64b3a70a8ac0d81efa7224caf070743bca8b29b96aa1b3f49844b8af63e1aed4", "seconds": 0.33857045881450176, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457894912, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-opposite", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.677699089050293, "actual_norm": 9.681968688964844, "relative_norm": 0.04999075058746375, "target_actual_norm": 9.682130813598633, "match_error": -0.0001621246337890625, "scalar": 9.677698685845826, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1810, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.00041067633363926727, -6.739181412414249e-09], "logp": -0.0004106830728206797, "cache_id": 1811, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.677699089050293, "actual_norm": 9.681968688964844, "relative_norm": 0.04999075058746375, "target_actual_norm": 9.682130813598633, "match_error": -0.0001621246337890625, "scalar": 9.677698685845826, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.62541067633364, -0.0001231513096875833], "logp": -16.625533827643327, "cache_id": 1812, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.677699089050293, "actual_norm": 9.681968688964844, "relative_norm": 0.04999075058746375, "target_actual_norm": 9.682130813598633, "match_error": -0.0001621246337890625, "scalar": 9.677698685845826, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.625123144570505, "first_log_odds": -16.625, "later_log_odds": -0.0001231445705052181, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-opposite.npz", "array_sha256": "6dec54cded709d4af2f77e446481a7df5a78ea1938f501350e1de37feec26f6d", "seconds": 0.336217500269413, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457894912, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-unrelated", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.688039779663086, "actual_norm": 9.683316230773926, "relative_norm": 0.04999770832804814, "target_actual_norm": 9.682130813598633, "match_error": 0.0011854171752929688, "scalar": 9.688040317269042, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1813, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0005892598917025721, -8.600912611771768e-09], "logp": -0.0005892684926151839, "cache_id": 1814, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.688039779663086, "actual_norm": 9.683316230773926, "relative_norm": 0.04999770832804814, "target_actual_norm": 9.682130813598633, "match_error": 0.0011854171752929688, "scalar": 9.688040317269042, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.625589259891703, -6.666911872836398e-05], "logp": -15.625655929010431, "cache_id": 1815, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.688039779663086, "actual_norm": 9.683316230773926, "relative_norm": 0.04999770832804814, "target_actual_norm": 9.682130813598633, "match_error": 0.0011854171752929688, "scalar": 9.688040317269042, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.625066660517817, "first_log_odds": -15.625, "later_log_odds": -6.666051781678561e-05, "first_differing_log_odds": -15.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-unrelated.npz", "array_sha256": "b198094724bdf5fed31cf40e5de716cd6cc1ec87c078134aa6b5c2807de75a96", "seconds": 0.33522283378988504, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457894912, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-ordinary", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.662556648254395, "actual_norm": 9.682463645935059, "relative_norm": 0.04999330619069326, "target_actual_norm": 9.682130813598633, "match_error": 0.00033283233642578125, "scalar": 9.662555582690402, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1816, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0005833256938278014, -6.518314755582415e-09], "logp": -0.000583332212142557, "cache_id": 1817, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.662556648254395, "actual_norm": 9.682463645935059, "relative_norm": 0.04999330619069326, "target_actual_norm": 9.682130813598633, "match_error": 0.00033283233642578125, "scalar": 9.662555582690402, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.250583325693828, -3.052641355008947e-05], "logp": -15.250613852107378, "cache_id": 1818, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.662556648254395, "actual_norm": 9.682463645935059, "relative_norm": 0.04999330619069326, "target_actual_norm": 9.682130813598633, "match_error": 0.00033283233642578125, "scalar": 9.662555582690402, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.250030519895235, "first_log_odds": -15.25, "later_log_odds": -3.051989523505938e-05, "first_differing_log_odds": -15.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-ordinary.npz", "array_sha256": "0f4f2cf2b64527b1ccaf161fcdabde2716d72f0e03301881cdfe96cc956e33e7", "seconds": 0.34035429218783975, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457927680, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-random0", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.668761253356934, "actual_norm": 9.681943893432617, "relative_norm": 0.04999062256109773, "target_actual_norm": 9.682130813598633, "match_error": -0.000186920166015625, "scalar": 9.668760561544332, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1819, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0003876179850281115, -7.2807843933003754e-09], "logp": -0.0003876252658125048, "cache_id": 1820, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.668761253356934, "actual_norm": 9.681943893432617, "relative_norm": 0.04999062256109773, "target_actual_norm": 9.682130813598633, "match_error": -0.000186920166015625, "scalar": 9.668760561544332, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.000387617985027, -7.168329683423089e-05], "logp": -16.00045930128186, "cache_id": 1821, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.668761253356934, "actual_norm": 9.681943893432617, "relative_norm": 0.04999062256109773, "target_actual_norm": 9.682130813598633, "match_error": -0.000186920166015625, "scalar": 9.668760561544332, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.00007167601605, "first_log_odds": -15.999999999999998, "later_log_odds": -7.167601605040375e-05, "first_differing_log_odds": -15.999999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-random0.npz", "array_sha256": "1732dd47b19433486be8e36d619650df4ddcb20a0fbd9511a1f9e2ed0e216261", "seconds": 0.33803129103034735, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457927680, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-random1", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.669351577758789, "actual_norm": 9.682210922241211, "relative_norm": 0.049992001306578, "target_actual_norm": 9.682130813598633, "match_error": 8.0108642578125e-05, "scalar": 9.669351511911373, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1822, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0004257334333055102, -5.526424413921251e-09], "logp": -0.0004257389597299241, "cache_id": 1823, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.669351577758789, "actual_norm": 9.682210922241211, "relative_norm": 0.049992001306578, "target_actual_norm": 9.682130813598633, "match_error": 8.0108642578125e-05, "scalar": 9.669351511911373, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.875425733433305, -5.236280902518808e-05], "logp": -15.87547809624233, "cache_id": 1824, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.669351577758789, "actual_norm": 9.682210922241211, "relative_norm": 0.049992001306578, "target_actual_norm": 9.682130813598633, "match_error": 8.0108642578125e-05, "scalar": 9.669351511911373, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.8750523572826, "first_log_odds": -15.875, "later_log_odds": -5.235728260011285e-05, "first_differing_log_odds": -15.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-random1.npz", "array_sha256": "fa6fc4a95a8de456d87088acc7f19e592e1f6035a0ef929a8faadc62d99846b4", "seconds": 0.33820925001055, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457927680, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-random2", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.672435760498047, "actual_norm": 9.682123184204102, "relative_norm": 0.04999154829020591, "target_actual_norm": 9.682130813598633, "match_error": -7.62939453125e-06, "scalar": 9.672435534139368, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1825, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0006062468365821231, -8.554853455678049e-09], "logp": -0.0006062553914355787, "cache_id": 1826, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.672435760498047, "actual_norm": 9.682123184204102, "relative_norm": 0.04999154829020591, "target_actual_norm": 9.682130813598633, "match_error": -7.62939453125e-06, "scalar": 9.672435534139368, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.750606246836583, -4.414801984703746e-05], "logp": -15.75065039485643, "cache_id": 1827, "trace": [{"sequence_length": 34, "positions": [8], "layer": 13, "residual_norm": 193.67520141601562, "strength_one_norm": 44.929691314697266, "target_shrink_scale": 0.21537794913834993, "requested_norm": 9.672435760498047, "actual_norm": 9.682123184204102, "relative_norm": 0.04999154829020591, "target_actual_norm": 9.682130813598633, "match_error": -7.62939453125e-06, "scalar": 9.672435534139368, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.750044139464995, "first_log_odds": -15.75, "later_log_odds": -4.413946499504107e-05, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-random2.npz", "array_sha256": "df97fd607092018dc4ef87ece0e1391f7c4574cddc7c440a2c5f2dcf8e9572e7", "seconds": 0.3356181248091161, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457927680, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-main-bridge", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JPN<|im_end|>", "output_text": "JPN", "output_ids": [41, 17896, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 739.8399658203125, 18.664701461791992, 141.73988342285156, 209.5064697265625, 118.12511444091797, 104.3729476928711, 101.79360961914062, 239.0401611328125, 39.59619903564453, 288.52703857421875, 47.522193908691406, 143.55723571777344, 64.18327331542969, 2.956850528717041, 93.93242645263672, 24.36406707763672, 84.57794189453125, 77.5759048461914, 41.3504524230957, 262.50445556640625, 317.0079650878906, 333.1654968261719, 366.6127014160156, 184.96095275878906, 106.40298461914062, 1231.2548828125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 739.8184814453125, 18.71773910522461, 141.7524871826172, 209.45172119140625, 118.1161880493164, 104.3719482421875, 101.81970977783203, 239.0630645751953, 39.600730895996094, 288.47821044921875, 47.55131912231445, 143.62953186035156, 64.2052230834961, 3.096041679382324, 93.93225860595703, 24.399322509765625, 84.57005310058594, 77.54895782470703, 41.3741455078125, 262.5287780761719, 317.04937744140625, 333.17724609375, 366.6137390136719, 185.00457763671875, 106.40435791015625, 1231.240478515625]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.6937811970710754, 0.019459547474980354, 0.13970252871513367, 0.19684532284736633, 0.11532309651374817, 0.10007210075855255, 0.08900109678506851, 0.23469124734401703, 0.03788803890347481, 0.2930452525615692, 0.04907597228884697, 0.1437104195356369, 0.06163479760289192, 0.002938699908554554, 0.07571565359830856, 0.02444556914269924, 0.09478841722011566, 0.08499839901924133, 0.04872401803731918, 0.26078909635543823, 0.3360341191291809, 0.2927066683769226, 0.3515545427799225, 0.19849339127540588, 0.10276859253644943, 1.118412971496582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1828, "cached": true, "scoring": {"normalized": "JPN", "bare_format": false, "malformed": true, "first_code": null, "all_codes": [], "currency_class": "none", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": null, "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-29.708140979076255, -5.937807561330784e-09], "logp": -29.70814098501406, "cache_id": 1829, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 739.8399658203125, 18.664701461791992, 141.73988342285156, 209.5064697265625, 118.12511444091797, 104.3729476928711, 101.79360961914062, 239.0401611328125, 39.59619903564453, 288.52703857421875, 47.522193908691406, 143.55723571777344, 64.18327331542969, 2.956850528717041, 93.93242645263672, 24.36406707763672, 84.57794189453125, 77.5759048461914, 41.3504524230957, 262.50445556640625, 317.0079650878906, 333.1654968261719, 366.6127014160156, 184.96095275878906, 106.40298461914062, 1231.2548828125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 739.8184814453125, 18.71773910522461, 141.7524871826172, 209.45172119140625, 118.1161880493164, 104.3719482421875, 101.81970977783203, 239.0630645751953, 39.600730895996094, 288.47821044921875, 47.55131912231445, 143.62953186035156, 64.2052230834961, 3.096041679382324, 93.93225860595703, 24.399322509765625, 84.57005310058594, 77.54895782470703, 41.3741455078125, 262.5287780761719, 317.04937744140625, 333.17724609375, 366.6137390136719, 185.00457763671875, 106.40435791015625, 1231.240478515625]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.6937811970710754, 0.019459547474980354, 0.13970252871513367, 0.19684532284736633, 0.11532309651374817, 0.10007210075855255, 0.08900109678506851, 0.23469124734401703, 0.03788803890347481, 0.2930452525615692, 0.04907597228884697, 0.1437104195356369, 0.06163479760289192, 0.002938699908554554, 0.07571565359830856, 0.02444556914269924, 0.09478841722011566, 0.08499839901924133, 0.04872401803731918, 0.26078909635543823, 0.3360341191291809, 0.2927066683769226, 0.3515545427799225, 0.19849339127540588, 0.10276859253644943, 1.118412971496582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-2.965953479076255, -8.332649438770713e-05], "logp": -2.9660368055706425, "cache_id": 1830, "trace": [{"sequence_length": 34, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 739.8399658203125, 18.664701461791992, 141.73988342285156, 209.5064697265625, 118.12511444091797, 104.3729476928711, 101.79360961914062, 239.0401611328125, 39.59619903564453, 288.52703857421875, 47.522193908691406, 143.55723571777344, 64.18327331542969, 2.956850528717041, 93.93242645263672, 24.36406707763672, 84.57794189453125, 77.5759048461914, 41.3504524230957, 262.50445556640625, 317.0079650878906, 333.1654968261719, 366.6127014160156, 184.96095275878906, 106.40298461914062, 1231.2548828125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 739.8184814453125, 18.71773910522461, 141.7524871826172, 209.45172119140625, 118.1161880493164, 104.3719482421875, 101.81970977783203, 239.0630645751953, 39.600730895996094, 288.47821044921875, 47.55131912231445, 143.62953186035156, 64.2052230834961, 3.096041679382324, 93.93225860595703, 24.399322509765625, 84.57005310058594, 77.54895782470703, 41.3741455078125, 262.5287780761719, 317.04937744140625, 333.17724609375, 366.6137390136719, 185.00457763671875, 106.40435791015625, 1231.240478515625]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.6937811970710754, 0.019459547474980354, 0.13970252871513367, 0.19684532284736633, 0.11532309651374817, 0.10007210075855255, 0.08900109678506851, 0.23469124734401703, 0.03788803890347481, 0.2930452525615692, 0.04907597228884697, 0.1437104195356369, 0.06163479760289192, 0.002938699908554554, 0.07571565359830856, 0.02444556914269924, 0.09478841722011566, 0.08499839901924133, 0.04872401803731918, 0.26078909635543823, 0.3360341191291809, 0.2927066683769226, 0.3515545427799225, 0.19849339127540588, 0.10276859253644943, 1.118412971496582]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 26.74210417944342, "first_log_odds": 26.7421875, "later_log_odds": -8.332055658044624e-05, "first_differing_log_odds": 26.7421875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-main-bridge.npz", "array_sha256": "38b8f4f39960636f743198b489437dcdef5c4a281228eda4d551131fab30b3b8", "seconds": 0.34069804195314646, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-baseline", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1831, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.773586679004513, -4.8858203399654174e-08], "logp": -3.773586727862716, "cache_id": 1832, "trace": []}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.273586679004513, -0.0010766851746286917], "logp": -11.274663364179142, "cache_id": 1833, "trace": []}, "log_odds": -7.5010766363164265, "first_log_odds": -7.5, "later_log_odds": -0.0010766363164265158, "first_differing_log_odds": -7.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-baseline.npz", "array_sha256": "96b071a66af2e761a84827be272923328c097b7be432386c8c90909b8df3e484", "seconds": 1.2275319583714008, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-noop", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.580580711364746, "match_error": -9.580580711364746, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1834, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.773586679004513, -4.8858203399654174e-08], "logp": -3.773586727862716, "cache_id": 1835, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.580580711364746, "match_error": -9.580580711364746, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.273586679004513, -0.0010766851746286917], "logp": -11.274663364179142, "cache_id": 1836, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.580580711364746, "match_error": -9.580580711364746, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.5010766363164265, "first_log_odds": -7.5, "later_log_odds": -0.0010766363164265158, "first_differing_log_odds": -7.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-noop.npz", "array_sha256": "8afca9cfafebca1b22dc947c29ad4342052fefa04b87b45a82d3d2d13157319a", "seconds": 1.2188399168662727, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-j", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.576465606689453, "actual_norm": 9.580580711364746, "relative_norm": 0.04999817545377948, "target_actual_norm": 9.580580711364746, "match_error": 0.0, "scalar": 0.21227105702426285, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1837, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-6.0026225775730335, -5.055156987357208e-08], "logp": -6.002622628124604, "cache_id": 1838, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.576465606689453, "actual_norm": 9.580580711364746, "relative_norm": 0.04999817545377948, "target_actual_norm": 9.580580711364746, "match_error": 0.0, "scalar": 0.21227105702426285, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.002622577573034, -0.0005729831130292719], "logp": -11.003195560686063, "cache_id": 1839, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.576465606689453, "actual_norm": 9.580580711364746, "relative_norm": 0.04999817545377948, "target_actual_norm": 9.580580711364746, "match_error": 0.0, "scalar": 0.21227105702426285, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.000572932561459, "first_log_odds": -5.0, "later_log_odds": -0.0005729325614591829, "first_differing_log_odds": -5.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-j.npz", "array_sha256": "7ddca298ee53de70d9c3f9b83fe4d9b5b699a599465b231260ac87ba87dceb04", "seconds": 1.2336391671560705, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-opposite", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.582921028137207, "actual_norm": 9.579672813415527, "relative_norm": 0.049993437406856564, "target_actual_norm": 9.580580711364746, "match_error": -0.00090789794921875, "scalar": 9.582919720327482, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1840, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.2883721665990273, -5.4277734166337974e-08], "logp": -3.2883722208767616, "cache_id": 1841, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.582921028137207, "actual_norm": 9.579672813415527, "relative_norm": 0.049993437406856564, "target_actual_norm": 9.580580711364746, "match_error": -0.00090789794921875, "scalar": 9.582919720327482, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.663372166599027, -0.0018830041116576987], "logp": -11.665255170710685, "cache_id": 1842, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.582921028137207, "actual_norm": 9.579672813415527, "relative_norm": 0.049993437406856564, "target_actual_norm": 9.580580711364746, "match_error": -0.00090789794921875, "scalar": 9.582919720327482, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.376882949833924, "first_log_odds": -8.375, "later_log_odds": -0.001882949833923675, "first_differing_log_odds": -8.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-opposite.npz", "array_sha256": "1c4882c48d502df430c6488347bedd2e7ce923e5b8cbcb46989459c9762840b3", "seconds": 1.2185282921418548, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-unrelated", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.57536792755127, "actual_norm": 9.580582618713379, "relative_norm": 0.049998185407659566, "target_actual_norm": 9.580580711364746, "match_error": 1.9073486328125e-06, "scalar": 9.575367051250055, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1843, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.7735418727553465, -5.123653148167211e-08], "logp": -3.773541923991878, "cache_id": 1844, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.57536792755127, "actual_norm": 9.580582618713379, "relative_norm": 0.049998185407659566, "target_actual_norm": 9.580580711364746, "match_error": 1.9073486328125e-06, "scalar": 9.575367051250055, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.273541872755347, -0.0009573707195667255], "logp": -11.274499243474914, "cache_id": 1845, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.57536792755127, "actual_norm": 9.580582618713379, "relative_norm": 0.049998185407659566, "target_actual_norm": 9.580580711364746, "match_error": 1.9073486328125e-06, "scalar": 9.575367051250055, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.500957319483035, "first_log_odds": -7.5, "later_log_odds": -0.000957319483035235, "first_differing_log_odds": -7.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-unrelated.npz", "array_sha256": "d83346cd729080827f0b32b50db3f122adc271e9b816755ad5991d382e7c98d7", "seconds": 1.2232976672239602, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-ordinary", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.559239387512207, "actual_norm": 9.580792427062988, "relative_norm": 0.04999928033446948, "target_actual_norm": 9.580580711364746, "match_error": 0.0002117156982421875, "scalar": 9.559237254579784, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1846, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.758870803832663, -5.058232837886935e-08], "logp": -4.758870854414991, "cache_id": 1847, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.559239387512207, "actual_norm": 9.580792427062988, "relative_norm": 0.04999928033446948, "target_actual_norm": 9.580580711364746, "match_error": 0.0002117156982421875, "scalar": 9.559237254579784, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.258870803832663, -0.0009649070066031105], "logp": -11.259835710839265, "cache_id": 1848, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.559239387512207, "actual_norm": 9.580792427062988, "relative_norm": 0.04999928033446948, "target_actual_norm": 9.580580711364746, "match_error": 0.0002117156982421875, "scalar": 9.559237254579784, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -6.500964856424274, "first_log_odds": -6.5, "later_log_odds": -0.0009648564242743518, "first_differing_log_odds": -6.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-ordinary.npz", "array_sha256": "c26ca06ffefac747f9a4f8437a304f1ed3018745eaaaa27f3d91c9b20b96473f", "seconds": 1.2148446249775589, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-random0", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.577676773071289, "actual_norm": 9.580765724182129, "relative_norm": 0.04999914098014822, "target_actual_norm": 9.580580711364746, "match_error": 0.0001850128173828125, "scalar": 9.577675223668848, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1849, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.018445440629955, -6.150090536458191e-08], "logp": -4.01844550213086, "cache_id": 1850, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.577676773071289, "actual_norm": 9.580765724182129, "relative_norm": 0.04999914098014822, "target_actual_norm": 9.580580711364746, "match_error": 0.0001850128173828125, "scalar": 9.577675223668848, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.268445440629955, -0.0009768245857532038], "logp": -11.26942226521571, "cache_id": 1851, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.577676773071289, "actual_norm": 9.580765724182129, "relative_norm": 0.04999914098014822, "target_actual_norm": 9.580580711364746, "match_error": 0.0001850128173828125, "scalar": 9.577675223668848, "random_seed": 20274008, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.250976763084849, "first_log_odds": -7.250000000000001, "later_log_odds": -0.0009767630848482867, "first_differing_log_odds": -7.250000000000001, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-random0.npz", "array_sha256": "10af909661e7743a8c5034aab0054db03ccf7b82feda2378a037e072fd825959", "seconds": 1.292683165986091, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-random1", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.575170516967773, "actual_norm": 9.580559730529785, "relative_norm": 0.04999806596109849, "target_actual_norm": 9.580580711364746, "match_error": -2.09808349609375e-05, "scalar": 9.57517175313842, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1852, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-3.0490547995164436, -5.689399078769643e-08], "logp": -3.0490548564104345, "cache_id": 1853, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.575170516967773, "actual_norm": 9.580559730529785, "relative_norm": 0.04999806596109849, "target_actual_norm": 9.580580711364746, "match_error": -2.09808349609375e-05, "scalar": 9.57517175313842, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.049054799516444, -0.0008703676851577521], "logp": -11.049925167201602, "cache_id": 1854, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.575170516967773, "actual_norm": 9.580559730529785, "relative_norm": 0.04999806596109849, "target_actual_norm": 9.580580711364746, "match_error": -2.09808349609375e-05, "scalar": 9.57517175313842, "random_seed": 20274009, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.000870310791168, "first_log_odds": -8.0, "later_log_odds": -0.0008703107911678387, "first_differing_log_odds": -8.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-random1.npz", "array_sha256": "949fba19321794b798a76176bc68c151296982268be5e56b51befb2842ae1331", "seconds": 1.2260806248523295, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-random2", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.565157890319824, "actual_norm": 9.580645561218262, "relative_norm": 0.04999851388570254, "target_actual_norm": 9.580580711364746, "match_error": 6.4849853515625e-05, "scalar": 9.565157871016709, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1855, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-4.264402690981753, -5.5955743805016827e-08], "logp": -4.264402746937497, "cache_id": 1856, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.565157890319824, "actual_norm": 9.580645561218262, "relative_norm": 0.04999851388570254, "target_actual_norm": 9.580580711364746, "match_error": 6.4849853515625e-05, "scalar": 9.565157871016709, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-11.639402690981752, -0.0008878585308009572], "logp": -11.640290549512553, "cache_id": 1857, "trace": [{"sequence_length": 31, "positions": [8], "layer": 13, "residual_norm": 191.6186065673828, "strength_one_norm": 45.11432647705078, "target_shrink_scale": 0.21227105702426285, "requested_norm": 9.565157890319824, "actual_norm": 9.580645561218262, "relative_norm": 0.04999851388570254, "target_actual_norm": 9.580580711364746, "match_error": 6.4849853515625e-05, "scalar": 9.565157871016709, "random_seed": 20274010, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.375887802575056, "first_log_odds": -7.374999999999999, "later_log_odds": -0.0008878025750567176, "first_differing_log_odds": -7.374999999999999, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-random2.npz", "array_sha256": "86ed44aff03d2cbf8f5d13553ea2a822d29230fcaceee489b9f0dd76e3a16780", "seconds": 1.2304296670481563, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-31-paraphrase-bridge", "case_id": "eval-31", "city": "Jakarta", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 63033, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 697.7620239257812, 177.0149688720703, 91.81169891357422, 129.0094757080078, 160.2213592529297, 89.65218353271484, 86.12429809570312, 227.2479705810547, 27.819326400756836, 282.67596435546875, 45.65851593017578, 130.31112670898438, 41.82814407348633, 117.48724365234375, 42.50495147705078, 90.10469055175781, 275.2423095703125, 318.4979248046875, 324.093505859375, 347.68084716796875, 154.8690643310547, 124.79182434082031, 863.7247314453125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 697.739501953125, 177.00234985351562, 91.79894256591797, 129.0100555419922, 160.18675231933594, 89.60786437988281, 86.18572998046875, 227.21412658691406, 27.888614654541016, 282.7060241699219, 45.67856216430664, 130.29356384277344, 41.85289001464844, 117.51677703857422, 42.510467529296875, 90.12699890136719, 275.2979736328125, 318.4779357910156, 324.08349609375, 347.6559753417969, 154.8287811279297, 124.7791519165039, 863.5729370117188]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.6523860096931458, 0.17282287776470184, 0.0843728706240654, 0.12189775705337524, 0.1472701132297516, 0.08590850979089737, 0.0738193690776825, 0.21564602851867676, 0.026340002194046974, 0.2849530279636383, 0.04664277285337448, 0.12834890186786652, 0.039654865860939026, 0.12245235592126846, 0.04803222417831421, 0.1042594313621521, 0.26969850063323975, 0.3388170599937439, 0.28405383229255676, 0.32949692010879517, 0.16427630186080933, 0.11977645009756088, 0.8466925024986267]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1858, "cached": true, "scoring": {"normalized": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Jakarta is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-33.313638367909306, -4.061308273813473e-08], "logp": -33.31363840852239, "cache_id": 1859, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 697.7620239257812, 177.0149688720703, 91.81169891357422, 129.0094757080078, 160.2213592529297, 89.65218353271484, 86.12429809570312, 227.2479705810547, 27.819326400756836, 282.67596435546875, 45.65851593017578, 130.31112670898438, 41.82814407348633, 117.48724365234375, 42.50495147705078, 90.10469055175781, 275.2423095703125, 318.4979248046875, 324.093505859375, 347.68084716796875, 154.8690643310547, 124.79182434082031, 863.7247314453125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 697.739501953125, 177.00234985351562, 91.79894256591797, 129.0100555419922, 160.18675231933594, 89.60786437988281, 86.18572998046875, 227.21412658691406, 27.888614654541016, 282.7060241699219, 45.67856216430664, 130.29356384277344, 41.85289001464844, 117.51677703857422, 42.510467529296875, 90.12699890136719, 275.2979736328125, 318.4779357910156, 324.08349609375, 347.6559753417969, 154.8287811279297, 124.7791519165039, 863.5729370117188]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.6523860096931458, 0.17282287776470184, 0.0843728706240654, 0.12189775705337524, 0.1472701132297516, 0.08590850979089737, 0.0738193690776825, 0.21564602851867676, 0.026340002194046974, 0.2849530279636383, 0.04664277285337448, 0.12834890186786652, 0.039654865860939026, 0.12245235592126846, 0.04803222417831421, 0.1042594313621521, 0.26969850063323975, 0.3388170599937439, 0.28405383229255676, 0.32949692010879517, 0.16427630186080933, 0.11977645009756088, 0.8466925024986267]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-6.2823883679093075, -0.001115097572867939], "logp": -6.283503465482175, "cache_id": 1860, "trace": [{"sequence_length": 31, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 697.7620239257812, 177.0149688720703, 91.81169891357422, 129.0094757080078, 160.2213592529297, 89.65218353271484, 86.12429809570312, 227.2479705810547, 27.819326400756836, 282.67596435546875, 45.65851593017578, 130.31112670898438, 41.82814407348633, 117.48724365234375, 42.50495147705078, 90.10469055175781, 275.2423095703125, 318.4979248046875, 324.093505859375, 347.68084716796875, 154.8690643310547, 124.79182434082031, 863.7247314453125]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 697.739501953125, 177.00234985351562, 91.79894256591797, 129.0100555419922, 160.18675231933594, 89.60786437988281, 86.18572998046875, 227.21412658691406, 27.888614654541016, 282.7060241699219, 45.67856216430664, 130.29356384277344, 41.85289001464844, 117.51677703857422, 42.510467529296875, 90.12699890136719, 275.2979736328125, 318.4779357910156, 324.08349609375, 347.6559753417969, 154.8287811279297, 124.7791519165039, 863.5729370117188]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.6523860096931458, 0.17282287776470184, 0.0843728706240654, 0.12189775705337524, 0.1472701132297516, 0.08590850979089737, 0.0738193690776825, 0.21564602851867676, 0.026340002194046974, 0.2849530279636383, 0.04664277285337448, 0.12834890186786652, 0.039654865860939026, 0.12245235592126846, 0.04803222417831421, 0.1042594313621521, 0.26969850063323975, 0.3388170599937439, 0.28405383229255676, 0.32949692010879517, 0.16427630186080933, 0.11977645009756088, 0.8466925024986267]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 27.03013494304021, "first_log_odds": 27.03125, "later_log_odds": -0.001115056959790195, "first_differing_log_odds": 27.03125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-31-paraphrase-bridge.npz", "array_sha256": "2565f4a863cf81b770734b1cf05ea2be9a62f5e90a1423fb6e90793f8785866d", "seconds": 1.225788583047688, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-baseline", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1861, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0009317306940249371, -1.0651291569748375e-08], "logp": -0.0009317413453165069, "cache_id": 1862, "trace": []}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.375931730694024, -1.5282405707307603e-06], "logp": -16.375933258934595, "cache_id": 1863, "trace": []}, "log_odds": -16.37500151758928, "first_log_odds": -16.375, "later_log_odds": -1.5175892791319257e-06, "first_differing_log_odds": -16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-baseline.npz", "array_sha256": "7af0100c94366cee3068b7e1e8a73015d74a1629917df28ed09ac5d0ca077a71", "seconds": 0.39415145898237824, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-noop", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.593036651611328, "match_error": -9.593036651611328, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1864, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0009317306940249371, -1.0651291569748375e-08], "logp": -0.0009317413453165069, "cache_id": 1865, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.593036651611328, "match_error": -9.593036651611328, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.375931730694024, -1.5282405707307603e-06], "logp": -16.375933258934595, "cache_id": 1866, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.593036651611328, "match_error": -9.593036651611328, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.37500151758928, "first_log_odds": -16.375, "later_log_odds": -1.5175892791319257e-06, "first_differing_log_odds": -16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-noop.npz", "array_sha256": "16954f5d171978cbffa18d33a086c091e37f77922832aeb5f8f1b708892c8601", "seconds": 0.33926008408889174, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-j", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.590256690979004, "actual_norm": 9.593036651611328, "relative_norm": 0.04999850884671422, "target_actual_norm": 9.593036651611328, "match_error": 0.0, "scalar": 0.25900052483266667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1867, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0011458143194747247, -1.2542757420629409e-08], "logp": -0.0011458268622321452, "cache_id": 1868, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.590256690979004, "actual_norm": 9.593036651611328, "relative_norm": 0.04999850884671422, "target_actual_norm": 9.593036651611328, "match_error": 0.0, "scalar": 0.25900052483266667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-15.751145814319475, -1.8691447580548415e-06], "logp": -15.751147683464232, "cache_id": 1869, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.590256690979004, "actual_norm": 9.593036651611328, "relative_norm": 0.04999850884671422, "target_actual_norm": 9.593036651611328, "match_error": 0.0, "scalar": 0.25900052483266667, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -15.750001856602001, "first_log_odds": -15.75, "later_log_odds": -1.8566020010268858e-06, "first_differing_log_odds": -15.75, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-j.npz", "array_sha256": "14a70d5c75ff3fa86e26b4409dee4ccc1ba9130de34070525f13e7d25832af7b", "seconds": 0.3368780417367816, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-opposite", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.584912300109863, "actual_norm": 9.593006134033203, "relative_norm": 0.04999834979036373, "target_actual_norm": 9.593036651611328, "match_error": -3.0517578125e-05, "scalar": 9.584912665799493, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1870, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0008946288431110865, -9.665206588326341e-09], "logp": -0.0008946385083176749, "cache_id": 1871, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.584912300109863, "actual_norm": 9.593006134033203, "relative_norm": 0.04999834979036373, "target_actual_norm": 9.593036651611328, "match_error": -3.0517578125e-05, "scalar": 9.584912665799493, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.62589462884311, -2.366974825723646e-06], "logp": -16.625896995817936, "cache_id": 1872, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.584912300109863, "actual_norm": 9.593006134033203, "relative_norm": 0.04999834979036373, "target_actual_norm": 9.593036651611328, "match_error": -3.0517578125e-05, "scalar": 9.584912665799493, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.62500235730962, "first_log_odds": -16.625, "later_log_odds": -2.3573096186169096e-06, "first_differing_log_odds": -16.625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-opposite.npz", "array_sha256": "fe2a77f34ce392d37281f2baa114f1293c4fa9f34bd9e027ddd689a9780387a3", "seconds": 0.3382204999215901, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-unrelated", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.591570854187012, "actual_norm": 9.593009948730469, "relative_norm": 0.04999836967240754, "target_actual_norm": 9.593036651611328, "match_error": -2.6702880859375e-05, "scalar": 9.59157287038397, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1873, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0008885059647568687, -1.4032637175939621e-08], "logp": -0.0008885199973940447, "cache_id": 1874, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.591570854187012, "actual_norm": 9.593009948730469, "relative_norm": 0.04999836967240754, "target_actual_norm": 9.593036651611328, "match_error": -2.6702880859375e-05, "scalar": 9.59157287038397, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.500888505964756, -1.811279927474348e-06], "logp": -16.500890317244682, "cache_id": 1875, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.591570854187012, "actual_norm": 9.593009948730469, "relative_norm": 0.04999836967240754, "target_actual_norm": 9.593036651611328, "match_error": -2.6702880859375e-05, "scalar": 9.59157287038397, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.50000179724729, "first_log_odds": -16.5, "later_log_odds": -1.7972472896587988e-06, "first_differing_log_odds": -16.5, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-unrelated.npz", "array_sha256": "762e73a4b3cbc781eabec22a00dc98ce771517b8881b47be411aa0fdc0f4c6a5", "seconds": 0.3391541657038033, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-ordinary", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.58531379699707, "actual_norm": 9.5931396484375, "relative_norm": 0.0499990456618971, "target_actual_norm": 9.593036651611328, "match_error": 0.000102996826171875, "scalar": 9.585315205637016, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1876, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0008192367018888136, -1.2500686629861063e-08], "logp": -0.0008192492025754434, "cache_id": 1877, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.58531379699707, "actual_norm": 9.5931396484375, "relative_norm": 0.0499990456618971, "target_actual_norm": 9.593036651611328, "match_error": 0.000102996826171875, "scalar": 9.585315205637016, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.37581923670189, -2.3875128596896845e-06], "logp": -16.37582162421475, "cache_id": 1878, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.58531379699707, "actual_norm": 9.5931396484375, "relative_norm": 0.0499990456618971, "target_actual_norm": 9.593036651611328, "match_error": 0.000102996826171875, "scalar": 9.585315205637016, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.375002375012176, "first_log_odds": -16.375, "later_log_odds": -2.375012176258906e-06, "first_differing_log_odds": -16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-ordinary.npz", "array_sha256": "78fdb38503faeda8eb151711d2b035a7081ad6f0c49ce1a754c0c93b3ed2e0e4", "seconds": 0.3393408749252558, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-random0", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.605040550231934, "actual_norm": 9.59307861328125, "relative_norm": 0.049998727549196126, "target_actual_norm": 9.593036651611328, "match_error": 4.1961669921875e-05, "scalar": 9.605039657675661, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1879, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0005032284481006393, -9.98076927231867e-09], "logp": -0.0005032384288699116, "cache_id": 1880, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.605040550231934, "actual_norm": 9.59307861328125, "relative_norm": 0.049998727549196126, "target_actual_norm": 9.593036651611328, "match_error": 4.1961669921875e-05, "scalar": 9.605039657675661, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.0005032284481, -2.0086612170483314e-06], "logp": -17.000505237109316, "cache_id": 1881, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.605040550231934, "actual_norm": 9.59307861328125, "relative_norm": 0.049998727549196126, "target_actual_norm": 9.593036651611328, "match_error": 4.1961669921875e-05, "scalar": 9.605039657675661, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -17.000001998680446, "first_log_odds": -17.0, "later_log_odds": -1.9986804460359053e-06, "first_differing_log_odds": -17.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-random0.npz", "array_sha256": "7e542ad1a8fa88b21a0f57a42e66aab703190d0115430e9f2ab0c6756549018f", "seconds": 0.34052849980071187, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-random1", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.587529182434082, "actual_norm": 9.593087196350098, "relative_norm": 0.0499987722837947, "target_actual_norm": 9.593036651611328, "match_error": 5.054473876953125e-05, "scalar": 9.587529174743395, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1882, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0010050637568947893, -1.1404738199852278e-08], "logp": -0.0010050751616329891, "cache_id": 1883, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.587529182434082, "actual_norm": 9.593087196350098, "relative_norm": 0.0499987722837947, "target_actual_norm": 9.593036651611328, "match_error": 5.054473876953125e-05, "scalar": 9.587529174743395, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.376005063756896, -1.4708779632653457e-06], "logp": -16.376006534634858, "cache_id": 1884, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.587529182434082, "actual_norm": 9.593087196350098, "relative_norm": 0.0499987722837947, "target_actual_norm": 9.593036651611328, "match_error": 5.054473876953125e-05, "scalar": 9.587529174743395, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.375001459473225, "first_log_odds": -16.375, "later_log_odds": -1.4594732249406661e-06, "first_differing_log_odds": -16.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-random1.npz", "array_sha256": "599a4f2c07bf88feddb60e33a36bb98417f3bdb6a8b858598ec0ed1b1eb44a3b", "seconds": 0.33809733437374234, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-random2", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "IDR<|im_end|>", "output_text": "IDR", "output_ids": [915, 49, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.577520370483398, "actual_norm": 9.592670440673828, "relative_norm": 0.04999660017050841, "target_actual_norm": 9.593036651611328, "match_error": -0.0003662109375, "scalar": 9.577520570601337, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1885, "cached": true, "scoring": {"normalized": "IDR", "bare_format": true, "malformed": false, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": true, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-0.0009369407492173476, -1.2188405765884724e-08], "logp": -0.0009369529376231135, "cache_id": 1886, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.577520370483398, "actual_norm": 9.592670440673828, "relative_norm": 0.04999660017050841, "target_actual_norm": 9.593036651611328, "match_error": -0.0003662109375, "scalar": 9.577520570601337, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-16.250936940749217, -1.7104258787599673e-06], "logp": -16.250938651175094, "cache_id": 1887, "trace": [{"sequence_length": 35, "positions": [9], "layer": 13, "residual_norm": 191.866455078125, "strength_one_norm": 37.0279426574707, "target_shrink_scale": 0.25900052483266667, "requested_norm": 9.577520370483398, "actual_norm": 9.592670440673828, "relative_norm": 0.04999660017050841, "target_actual_norm": 9.593036651611328, "match_error": -0.0003662109375, "scalar": 9.577520570601337, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -16.250001698237472, "first_log_odds": -16.25, "later_log_odds": -1.6982374724250349e-06, "first_differing_log_odds": -16.25, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-random2.npz", "array_sha256": "57e329a38d52e5945a86aa7ff65c37f928258252025885d2c2995d7fcfd0c8bf", "seconds": 0.33872495917603374, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-main-bridge", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "main", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "JOD<|im_end|>", "output_text": "JOD", "output_ids": [41, 2069, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 8.458942413330078, 683.2649536132812, 20.269912719726562, 163.3966827392578, 208.91140747070312, 118.26991271972656, 100.0267333984375, 112.65457153320312, 254.70960998535156, 28.33518409729004, 293.4349060058594, 47.12952423095703, 145.25747680664062, 64.33857727050781, 8.117313385009766, 80.41088104248047, 14.69392204284668, 69.9009017944336, 78.85575866699219, 46.75737762451172, 256.1452941894531, 309.3631896972656, 320.1079406738281, 354.38092041015625, 176.07723999023438, 110.0892562866211, 1147.8240966796875]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 8.631393432617188, 683.1544189453125, 20.263378143310547, 163.41314697265625, 208.91673278808594, 118.27054595947266, 99.9951400756836, 112.62311553955078, 254.74668884277344, 28.36343765258789, 293.42840576171875, 47.14152526855469, 145.27139282226562, 64.40482330322266, 8.174336433410645, 80.39417266845703, 14.696670532226562, 69.91861724853516, 78.85386657714844, 46.742496490478516, 256.1080017089844, 309.4218444824219, 320.0276794433594, 354.3775329589844, 176.08595275878906, 110.03931427001953, 1147.8133544921875]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.006889676675200462, 0.6811680793762207, 0.021036367863416672, 0.16094695031642914, 0.19467154145240784, 0.11509483307600021, 0.09601619839668274, 0.0978858694434166, 0.2476126253604889, 0.027040213346481323, 0.29903990030288696, 0.04871075972914696, 0.14579784870147705, 0.06129446625709534, 0.007690145634114742, 0.0649460107088089, 0.014635010622441769, 0.07818342000246048, 0.08704276382923126, 0.055730078369379044, 0.2558009922504425, 0.326255202293396, 0.28341567516326904, 0.3433230221271515, 0.19084423780441284, 0.10602941364049911, 1.0901790857315063]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1888, "cached": true, "scoring": {"normalized": "JOD", "bare_format": true, "malformed": false, "first_code": "JOD", "all_codes": ["JOD"], "currency_class": "other", "correct_code": false, "clean_source": false, "clean_target": false, "prefix_before_code": "", "substantive_prefix_before_code": false, "country_mentions": [], "country_before_code": false, "country_before_currency_marker": false, "substantive_prefix_before_currency_marker": false}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-29.765739649209216, -7.830436712803308e-09], "logp": -29.765739657039653, "cache_id": 1889, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 8.458942413330078, 683.2649536132812, 20.269912719726562, 163.3966827392578, 208.91140747070312, 118.26991271972656, 100.0267333984375, 112.65457153320312, 254.70960998535156, 28.33518409729004, 293.4349060058594, 47.12952423095703, 145.25747680664062, 64.33857727050781, 8.117313385009766, 80.41088104248047, 14.69392204284668, 69.9009017944336, 78.85575866699219, 46.75737762451172, 256.1452941894531, 309.3631896972656, 320.1079406738281, 354.38092041015625, 176.07723999023438, 110.0892562866211, 1147.8240966796875]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 8.631393432617188, 683.1544189453125, 20.263378143310547, 163.41314697265625, 208.91673278808594, 118.27054595947266, 99.9951400756836, 112.62311553955078, 254.74668884277344, 28.36343765258789, 293.42840576171875, 47.14152526855469, 145.27139282226562, 64.40482330322266, 8.174336433410645, 80.39417266845703, 14.696670532226562, 69.91861724853516, 78.85386657714844, 46.742496490478516, 256.1080017089844, 309.4218444824219, 320.0276794433594, 354.3775329589844, 176.08595275878906, 110.03931427001953, 1147.8133544921875]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.006889676675200462, 0.6811680793762207, 0.021036367863416672, 0.16094695031642914, 0.19467154145240784, 0.11509483307600021, 0.09601619839668274, 0.0978858694434166, 0.2476126253604889, 0.027040213346481323, 0.29903990030288696, 0.04871075972914696, 0.14579784870147705, 0.06129446625709534, 0.007690145634114742, 0.0649460107088089, 0.014635010622441769, 0.07818342000246048, 0.08704276382923126, 0.055730078369379044, 0.2558009922504425, 0.326255202293396, 0.28341567516326904, 0.3433230221271515, 0.19084423780441284, 0.10602941364049911, 1.0901790857315063]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-2.0430833992092174, -1.6713674725694644e-06], "logp": -2.04308507057669, "cache_id": 1890, "trace": [{"sequence_length": 35, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 48.79662322998047, 80.52522277832031, 141.2006072998047, 159.5072479248047, 68.0926284790039, 8.458942413330078, 683.2649536132812, 20.269912719726562, 163.3966827392578, 208.91140747070312, 118.26991271972656, 100.0267333984375, 112.65457153320312, 254.70960998535156, 28.33518409729004, 293.4349060058594, 47.12952423095703, 145.25747680664062, 64.33857727050781, 8.117313385009766, 80.41088104248047, 14.69392204284668, 69.9009017944336, 78.85575866699219, 46.75737762451172, 256.1452941894531, 309.3631896972656, 320.1079406738281, 354.38092041015625, 176.07723999023438, 110.0892562866211, 1147.8240966796875]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 48.781131744384766, 80.5718994140625, 141.24191284179688, 159.52149963378906, 68.12522888183594, 8.631393432617188, 683.1544189453125, 20.263378143310547, 163.41314697265625, 208.91673278808594, 118.27054595947266, 99.9951400756836, 112.62311553955078, 254.74668884277344, 28.36343765258789, 293.42840576171875, 47.14152526855469, 145.27139282226562, 64.40482330322266, 8.174336433410645, 80.39417266845703, 14.696670532226562, 69.91861724853516, 78.85386657714844, 46.742496490478516, 256.1080017089844, 309.4218444824219, 320.0276794433594, 354.3775329589844, 176.08595275878906, 110.03931427001953, 1147.8133544921875]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.04477936401963234, 0.070692278444767, 0.11490778625011444, 0.13653768599033356, 0.06003200262784958, 0.006889676675200462, 0.6811680793762207, 0.021036367863416672, 0.16094695031642914, 0.19467154145240784, 0.11509483307600021, 0.09601619839668274, 0.0978858694434166, 0.2476126253604889, 0.027040213346481323, 0.29903990030288696, 0.04871075972914696, 0.14579784870147705, 0.06129446625709534, 0.007690145634114742, 0.0649460107088089, 0.014635010622441769, 0.07818342000246048, 0.08704276382923126, 0.055730078369379044, 0.2558009922504425, 0.326255202293396, 0.28341567516326904, 0.3433230221271515, 0.19084423780441284, 0.10602941364049911, 1.0901790857315063]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 27.722654586462962, "first_log_odds": 27.72265625, "later_log_odds": -1.663537037899232e-06, "first_differing_log_odds": 27.72265625, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-main-bridge.npz", "array_sha256": "b85ab6345cbae0d08019058d6766c05a76c64be1490b1d308164088ee534bda3", "seconds": 0.33699312480166554, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-baseline", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "baseline", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [], "cache_id": 1891, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-10.500038269120976, -1.5175678072871263e-07], "logp": -10.500038420877758, "cache_id": 1892, "trace": []}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.625038269120978, -5.2736452952801946e-05], "logp": -17.625091005573932, "cache_id": 1893, "trace": []}, "log_odds": -7.125052584696174, "first_log_odds": -7.125000000000002, "later_log_odds": -5.25846961725307e-05, "first_differing_log_odds": -7.125000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-baseline.npz", "array_sha256": "5f9e837edbed71a9ebb458bc27930b16eedf0b1547b8bd1f68e4fe4fc7b0e8f7", "seconds": 1.267532791942358, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-noop", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "noop", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.60274887084961, "match_error": -9.60274887084961, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1894, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-10.500038269120976, -1.5175678072871263e-07], "logp": -10.500038420877758, "cache_id": 1895, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.60274887084961, "match_error": -9.60274887084961, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.625038269120978, -5.2736452952801946e-05], "logp": -17.625091005573932, "cache_id": 1896, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 0.0, "actual_norm": 0.0, "relative_norm": 0.0, "target_actual_norm": 9.60274887084961, "match_error": -9.60274887084961, "scalar": 0.0, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.125052584696174, "first_log_odds": -7.125000000000002, "later_log_odds": -5.25846961725307e-05, "first_differing_log_odds": -7.125000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-noop.npz", "array_sha256": "a13ae94b5ae141634961472d48f9897e0ad2e38f7debc667e80b66741b4f5fd8", "seconds": 1.2564781666733325, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-j", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "j", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.594615936279297, "actual_norm": 9.60274887084961, "relative_norm": 0.04999999205499027, "target_actual_norm": 9.60274887084961, "match_error": 0.0, "scalar": 0.26475228365717945, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1897, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-11.37502147770747, -2.1703401985190082e-07], "logp": -11.37502169474149, "cache_id": 1898, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.594615936279297, "actual_norm": 9.60274887084961, "relative_norm": 0.04999999205499027, "target_actual_norm": 9.60274887084961, "match_error": 0.0, "scalar": 0.26475228365717945, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.25002147770747, -2.5800237195676827e-05], "logp": -17.250047277944667, "cache_id": 1899, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.594615936279297, "actual_norm": 9.60274887084961, "relative_norm": 0.04999999205499027, "target_actual_norm": 9.60274887084961, "match_error": 0.0, "scalar": 0.26475228365717945, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -5.875025583203177, "first_log_odds": -5.875, "later_log_odds": -2.5583203177248492e-05, "first_differing_log_odds": -5.875, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-j.npz", "array_sha256": "c30c4fa7d2bba5d3f6ccba364654bd1cc083501050b3c406a360d0ef4e74a704", "seconds": 1.2177019999362528, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-opposite", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "opposite", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.599526405334473, "actual_norm": 9.602743148803711, "relative_norm": 0.0499999622612038, "target_actual_norm": 9.60274887084961, "match_error": -5.7220458984375e-06, "scalar": 9.59952529182192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1900, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-9.875065422869179, -1.1719897609511804e-07], "logp": -9.875065540068155, "cache_id": 1901, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.599526405334473, "actual_norm": 9.602743148803711, "relative_norm": 0.0499999622612038, "target_actual_norm": 9.60274887084961, "match_error": -5.7220458984375e-06, "scalar": 9.59952529182192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-18.000065422869177, -6.88957883385231e-05], "logp": -18.000134318657516, "cache_id": 1902, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.599526405334473, "actual_norm": 9.602743148803711, "relative_norm": 0.0499999622612038, "target_actual_norm": 9.60274887084961, "match_error": -5.7220458984375e-06, "scalar": 9.59952529182192, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -8.12506877858936, "first_log_odds": -8.124999999999998, "later_log_odds": -6.877858936249481e-05, "first_differing_log_odds": -8.124999999999998, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-opposite.npz", "array_sha256": "bd17415f8aea9b8a77f9d842bc0ed8cadd1b8e9301e74d322f9e73b08b5ffb05", "seconds": 1.2070522499270737, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-unrelated", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "unrelated", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.608022689819336, "actual_norm": 9.602407455444336, "relative_norm": 0.04999821435906375, "target_actual_norm": 9.60274887084961, "match_error": -0.0003414154052734375, "scalar": 9.608023818349466, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1903, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-11.750012846372144, -1.9435978691731304e-07], "logp": -11.75001304073193, "cache_id": 1904, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.608022689819336, "actual_norm": 9.602407455444336, "relative_norm": 0.04999821435906375, "target_actual_norm": 9.60274887084961, "match_error": -0.0003414154052734375, "scalar": 9.608023818349466, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-18.750012846372144, -5.588065508359765e-05], "logp": -18.750068727027227, "cache_id": 1905, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.608022689819336, "actual_norm": 9.602407455444336, "relative_norm": 0.04999821435906375, "target_actual_norm": 9.60274887084961, "match_error": -0.0003414154052734375, "scalar": 9.608023818349466, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.000055686295296, "first_log_odds": -7.0, "later_log_odds": -5.568629529584257e-05, "first_differing_log_odds": -7.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-unrelated.npz", "array_sha256": "847789d6e692df3ac35d547ef047ddff8dcebf9b09cab5f965610ade6eb59d97", "seconds": 1.153209375217557, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-ordinary", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "ordinary", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.583738327026367, "actual_norm": 9.602581977844238, "relative_norm": 0.04999912306955133, "target_actual_norm": 9.60274887084961, "match_error": -0.00016689300537109375, "scalar": 9.58373708090221, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1906, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-10.625036596704765, -1.7972723692919e-07], "logp": -10.625036776432003, "cache_id": 1907, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.583738327026367, "actual_norm": 9.602581977844238, "relative_norm": 0.04999912306955133, "target_actual_norm": 9.60274887084961, "match_error": -0.00016689300537109375, "scalar": 9.58373708090221, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.750036596704767, -4.022994748702568e-05], "logp": -17.750076826652254, "cache_id": 1908, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.583738327026367, "actual_norm": 9.602581977844238, "relative_norm": 0.04999912306955133, "target_actual_norm": 9.60274887084961, "match_error": -0.00016689300537109375, "scalar": 9.58373708090221, "random_seed": null, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.125040050220251, "first_log_odds": -7.125000000000002, "later_log_odds": -4.0050220249554513e-05, "first_differing_log_odds": -7.125000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-ordinary.npz", "array_sha256": "fb6c50687a6f78a81cb25591c970ee94171680836f66344773ab2445166fb58a", "seconds": 1.326346917077899, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-random0", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random0", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.613443374633789, "actual_norm": 9.602731704711914, "relative_norm": 0.04999990267363084, "target_actual_norm": 9.60274887084961, "match_error": -1.71661376953125e-05, "scalar": 9.61344185795815, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1909, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-10.250048376446543, -1.2050067128631394e-07], "logp": -10.250048496947214, "cache_id": 1910, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.613443374633789, "actual_norm": 9.602731704711914, "relative_norm": 0.04999990267363084, "target_actual_norm": 9.60274887084961, "match_error": -1.71661376953125e-05, "scalar": 9.61344185795815, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.625048376446543, -3.961216148023067e-05], "logp": -17.625087988608023, "cache_id": 1911, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.613443374633789, "actual_norm": 9.602731704711914, "relative_norm": 0.04999990267363084, "target_actual_norm": 9.60274887084961, "match_error": -1.71661376953125e-05, "scalar": 9.61344185795815, "random_seed": 20274108, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.375039491660809, "first_log_odds": -7.375, "later_log_odds": -3.9491660809076734e-05, "first_differing_log_odds": -7.375, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-random0.npz", "array_sha256": "566a01913cddf97ef7994ff68336fbed0773394aa4e18cb1ed0ed802ae68ab97", "seconds": 1.278737916611135, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-random1", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random1", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.590150833129883, "actual_norm": 9.602482795715332, "relative_norm": 0.049998606643919044, "target_actual_norm": 9.60274887084961, "match_error": -0.00026607513427734375, "scalar": 9.59014760737773, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1912, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-11.125023675961788, -1.5267001652102723e-07], "logp": -11.125023828631806, "cache_id": 1913, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.590150833129883, "actual_norm": 9.602482795715332, "relative_norm": 0.049998606643919044, "target_actual_norm": 9.60274887084961, "match_error": -0.00026607513427734375, "scalar": 9.59014760737773, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-18.12502367596179, -4.906235079202849e-05], "logp": -18.125072738312582, "cache_id": 1914, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.590150833129883, "actual_norm": 9.602482795715332, "relative_norm": 0.049998606643919044, "target_actual_norm": 9.60274887084961, "match_error": -0.00026607513427734375, "scalar": 9.59014760737773, "random_seed": 20274109, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.0000489096807765, "first_log_odds": -7.0, "later_log_odds": -4.890968077653213e-05, "first_differing_log_odds": -7.0, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-random1.npz", "array_sha256": "be309c2bcf43b1ce608d6566f8182374e9fc775bada940d028e665ea1bb3127f", "seconds": 1.2692362922243774, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-random2", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "random2", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 58829, 431, 454, 19310, 11, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.584872245788574, "actual_norm": 9.602705001831055, "relative_norm": 0.049999763635960606, "target_actual_norm": 9.60274887084961, "match_error": -4.38690185546875e-05, "scalar": 9.584872659877874, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1915, "cached": true, "scoring": {"normalized": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the Indonesian Rupiah, and its three-letter ISO 4217 code is **", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 34, "end": 44}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-10.250046850222953, -1.69351286642682e-07], "logp": -10.25004701957424, "cache_id": 1916, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.584872245788574, "actual_norm": 9.602705001831055, "relative_norm": 0.049999763635960606, "target_actual_norm": 9.60274887084961, "match_error": -4.38690185546875e-05, "scalar": 9.584872659877874, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-17.625046850222954, -4.179820460780279e-05], "logp": -17.625088648427564, "cache_id": 1917, "trace": [{"sequence_length": 32, "positions": [9], "layer": 13, "residual_norm": 192.0550079345703, "strength_one_norm": 36.23997497558594, "target_shrink_scale": 0.26475228365717945, "requested_norm": 9.584872245788574, "actual_norm": 9.602705001831055, "relative_norm": 0.049999763635960606, "target_actual_norm": 9.60274887084961, "match_error": -4.38690185546875e-05, "scalar": 9.584872659877874, "random_seed": 20274110, "dtype": "torch.bfloat16", "device": "mps:0"}, {"sequence_length": 1, "positions": []}]}, "log_odds": -7.3750416288533245, "first_log_odds": -7.375000000000002, "later_log_odds": -4.162885332270605e-05, "first_differing_log_odds": -7.375000000000002, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-random2.npz", "array_sha256": "cab9328635a84ac9181f0df6248184069861d0344dc6d761f4191f2a3c814021", "seconds": 1.206434624735266, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}
{"key": "eval-32-paraphrase-bridge", "case_id": "eval-32", "city": "Medan", "country": "Indonesia", "code": "IDR", "target_country": "Japan", "target_code": "JPY", "template_id": "paraphrase", "condition": "bridge", "config": {"layer": 13, "position": "city", "cap": 0.05}, "output": "The currency used in Medan is the **Indonesian Rupiah** (IDR), and its three-letter ISO 4217 code is **IDR**.<|im_end|>", "output_text": "The currency used in Medan is the **Indonesian Rupiah** (IDR), and its three-letter ISO 4217 code is **IDR**.", "output_ids": [785, 11413, 1483, 304, 3264, 276, 374, 279, 3070, 1425, 3154, 1103, 431, 454, 19310, 334, 320, 915, 49, 701, 323, 1181, 2326, 79368, 21940, 220, 19, 17, 16, 22, 2038, 374, 3070, 915, 49, 334, 13, 151645], "eos_reached": true, "truncated": false, "generation_trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 23.801816940307617, 619.7488403320312, 224.93911743164062, 83.6789779663086, 291.4444885253906, 121.92757415771484, 87.80235290527344, 88.72770690917969, 238.2188262939453, 25.79340171813965, 287.7377014160156, 45.1988410949707, 132.3156280517578, 37.96992492675781, 122.72327423095703, 46.19042205810547, 92.53112030029297, 279.6091003417969, 319.8789978027344, 301.19122314453125, 347.652099609375, 138.8982391357422, 121.02703857421875, 682.1761474609375]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 23.860559463500977, 619.8281860351562, 224.93226623535156, 83.67322540283203, 291.3821105957031, 121.92498016357422, 87.78486633300781, 88.73099517822266, 238.25909423828125, 25.851842880249023, 287.7497863769531, 45.212947845458984, 132.29273986816406, 37.977027893066406, 122.76264190673828, 46.21714782714844, 92.52803802490234, 279.6219482421875, 319.93743896484375, 301.205078125, 347.6920166015625, 138.9239044189453, 121.0523681640625, 682.11328125]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.018606076017022133, 0.6096446514129639, 0.22387616336345673, 0.07681792229413986, 0.2711731791496277, 0.11227773129940033, 0.08487964421510696, 0.07538051903247833, 0.22281208634376526, 0.024301854893565178, 0.29209089279174805, 0.04656108468770981, 0.1297091394662857, 0.03585496544837952, 0.1308852732181549, 0.05219133198261261, 0.10579470545053482, 0.2754058539867401, 0.3374202847480774, 0.26424261927604675, 0.3308338224887848, 0.14681436121463776, 0.11650881171226501, 0.7047277688980103]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}, {"sequence_length": 1, "positions": []}], "cache_id": 1918, "cached": true, "scoring": {"normalized": "The currency used in Medan is the **Indonesian Rupiah** (IDR), and its three-letter ISO 4217 code is **IDR**.", "bare_format": false, "malformed": true, "first_code": "IDR", "all_codes": ["IDR", "IDR"], "currency_class": "source", "correct_code": true, "clean_source": false, "clean_target": false, "prefix_before_code": "The currency used in Medan is the **Indonesian Rupiah** (", "substantive_prefix_before_code": true, "country_mentions": [{"text": "Indonesian", "start": 36, "end": 46}], "country_before_code": true, "country_before_currency_marker": true, "substantive_prefix_before_currency_marker": true}, "source": {"ids": [915, 49], "tokens": ["ID", "R"], "log_probs": [-34.50065243756288, -1.24439720112852e-07], "logp": -34.5006525620026, "cache_id": 1919, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 23.801816940307617, 619.7488403320312, 224.93911743164062, 83.6789779663086, 291.4444885253906, 121.92757415771484, 87.80235290527344, 88.72770690917969, 238.2188262939453, 25.79340171813965, 287.7377014160156, 45.1988410949707, 132.3156280517578, 37.96992492675781, 122.72327423095703, 46.19042205810547, 92.53112030029297, 279.6091003417969, 319.8789978027344, 301.19122314453125, 347.652099609375, 138.8982391357422, 121.02703857421875, 682.1761474609375]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 23.860559463500977, 619.8281860351562, 224.93226623535156, 83.67322540283203, 291.3821105957031, 121.92498016357422, 87.78486633300781, 88.73099517822266, 238.25909423828125, 25.851842880249023, 287.7497863769531, 45.212947845458984, 132.29273986816406, 37.977027893066406, 122.76264190673828, 46.21714782714844, 92.52803802490234, 279.6219482421875, 319.93743896484375, 301.205078125, 347.6920166015625, 138.9239044189453, 121.0523681640625, 682.11328125]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.018606076017022133, 0.6096446514129639, 0.22387616336345673, 0.07681792229413986, 0.2711731791496277, 0.11227773129940033, 0.08487964421510696, 0.07538051903247833, 0.22281208634376526, 0.024301854893565178, 0.29209089279174805, 0.04656108468770981, 0.1297091394662857, 0.03585496544837952, 0.1308852732181549, 0.05219133198261261, 0.10579470545053482, 0.2754058539867401, 0.3374202847480774, 0.26424261927604675, 0.3308338224887848, 0.14681436121463776, 0.11650881171226501, 0.7047277688980103]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "target": {"ids": [27188, 56], "tokens": ["JP", "Y"], "log_probs": [-9.375652437562879, -4.346017717841583e-05], "logp": -9.375695897740057, "cache_id": 1920, "trace": [{"sequence_length": 32, "positions": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "requested_norm": [[320.3326110839844, 113.1124496459961, 147.4876708984375, 66.28754425048828, 98.79774475097656, 84.61792755126953, 117.92161560058594, 62.72513198852539, 23.801816940307617, 619.7488403320312, 224.93911743164062, 83.6789779663086, 291.4444885253906, 121.92757415771484, 87.80235290527344, 88.72770690917969, 238.2188262939453, 25.79340171813965, 287.7377014160156, 45.1988410949707, 132.3156280517578, 37.96992492675781, 122.72327423095703, 46.19042205810547, 92.53112030029297, 279.6091003417969, 319.8789978027344, 301.19122314453125, 347.652099609375, 138.8982391357422, 121.02703857421875, 682.1761474609375]], "actual_norm": [[320.2558288574219, 113.0596923828125, 147.4983673095703, 66.31813049316406, 98.80052185058594, 84.6545639038086, 117.886962890625, 62.7646484375, 23.860559463500977, 619.8281860351562, 224.93226623535156, 83.67322540283203, 291.3821105957031, 121.92498016357422, 87.78486633300781, 88.73099517822266, 238.25909423828125, 25.851842880249023, 287.7497863769531, 45.212947845458984, 132.29273986816406, 37.977027893066406, 122.76264190673828, 46.21714782714844, 92.52803802490234, 279.6219482421875, 319.93743896484375, 301.205078125, 347.6920166015625, 138.9239044189453, 121.0523681640625, 682.11328125]], "relative_norm": [[0.018691690638661385, 0.11993132531642914, 0.1631791740655899, 0.05782247707247734, 0.08993566781282425, 0.07354246079921722, 0.10484733432531357, 0.05575422942638397, 0.018606076017022133, 0.6096446514129639, 0.22387616336345673, 0.07681792229413986, 0.2711731791496277, 0.11227773129940033, 0.08487964421510696, 0.07538051903247833, 0.22281208634376526, 0.024301854893565178, 0.29209089279174805, 0.04656108468770981, 0.1297091394662857, 0.03585496544837952, 0.1308852732181549, 0.05219133198261261, 0.10579470545053482, 0.2754058539867401, 0.3374202847480774, 0.26424261927604675, 0.3308338224887848, 0.14681436121463776, 0.11650881171226501, 0.7047277688980103]], "device": "mps:0", "dtype": "torch.bfloat16"}, {"sequence_length": 1, "positions": []}]}, "log_odds": 25.12495666426254, "first_log_odds": 25.125, "later_log_odds": -4.3335737458960466e-05, "first_differing_log_odds": 25.125, "first_differing_index": 0, "shared_prefix_ids": [], "shared_prefix_text": "", "array_path": "results/arrays/eval-32-paraphrase-bridge.npz", "array_sha256": "beef53a32cf040320b893884358530c4d76816f3a7364e8f408ddffa4e182eb1", "seconds": 1.4743746249005198, "memory": {"max_rss_bytes": 1740980224, "mps_allocated_bytes": 3457960448, "mps_driver_bytes": 4002103296}}