{ "format": "renigma.head_fitting_replay", "version": 1, "status": "passed", "started_at": "2026-10-03T18:52:38.953256+00:00", "run": "G:\\RenigmaLLM_Artifacts\\head-fitting-runs\\rep-8d3baca72c53", "source": { "sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573", "git_commit": "d56d2ec3308539866e48bd441ae618d12fd5319d", "files": { "src/renigma/__init__.py": "d0622b66407dc62610ca3eb83b960807ea26f67384f832c75f35362db73280b3", "src/renigma/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d", "src/renigma/cli.py": "1d8f53bfeec7598e8d35f588fa0edddd7108a65e99f63703cee334d47746d7d3", "src/renigma/diagnostic_worker.py": "cadaa9cc4509df8bfd73212f9bd4e52000872d76cb8930deb3673c99f1542b2f", "src/renigma/evidence.py": "06f241371dfb3820267f83446396315ad21a65019b374be594be83c9d1ae51fa", "src/renigma/experiments.py": "cb7ec032bbb2e4e81982c7417d2e42dc34b7a26d3bbca6d60889675ec2f0f418", "src/renigma/gpu.py": "d2adc63a6de64be7d90a8f20298ce7c40326fa5972782cfc83adbb8d8fa92f95", "src/renigma/head_fitting.py": "e9f78f9be7b8037f0b869756473418afd8fca6a7bd16c85f701f3161d2d8704b", "src/renigma/head_fitting_study.py": "7d5e31e262173fdeebb5eae2cd5f446ebbd34d2853d7fe0b5863addb6f2e0f69", "src/renigma/head_intervention.py": "ef4abd910ece305f4c02dd4aa236d48fd9ba5f19d36c02d26548d6bceba1754f", "src/renigma/head_intervention_study.py": "6f13b7b29c3386a4dee18c637eeb6807b9e62f1f78a3e792a38850137186b40a", "src/renigma/math_checks.py": "02a036088fe46180a74cc1b1f89ffbce9c4d49b670786c6939a04c4f856bcb67", "src/renigma/memory.py": "a42b4424e82afbbec5b2b32982e221b20f64f27f6dfdf41ddeda9d600862151a", "src/renigma/memory_diagnostics.py": "1e45bb1c083b496d21891e8470fd1355b94d53d12010fc3fc27b4217abe9a605", "src/renigma/memory_experiments.py": "426bd7ee4c9286119d51f29bf6120a5841031ad03af374e9a113568d8c2486f5", "src/renigma/memory_language.py": "b2f04d3357f83e0556e3846f2e8f89e4b3e8bfc137bf7f0e3262f7e865126a6e", "src/renigma/memory_language_v2.py": "b42047f2ee10abc39209d8b7e59e4039738d6a6c2a26d71923e74881d8a6d14b", "src/renigma/memory_performance.py": "488dcf81a2bc1e2533de79993a26bac679c5b000e4a031281c40018a867662fd", "src/renigma/memory_scan.py": "fe117844a25c369b484a333790b4cee99f6362d0ddf59dab7160031b362e0952", "src/renigma/memory_scan_performance.py": "a09a63dba6860d92ed6f4bdf3f2b7a1796edb963823dc67810853158a3818c77", "src/renigma/memory_training.py": "60fa357f0131c9af7d4a36e744f3d8f23670b9830ba8c6f99a38557e7e87f9f8", "src/renigma/memory_worker.py": "86611aa159b6e7576995930c28e6352d7ddda3703e2bfa0c1c7706cb1c3b35df", "src/renigma/planner.py": "18c3cae774b0b9c6aead82997a29782432ff47e58746149cefa07b7d37c7c209", "src/renigma/representation.py": "426857a82c5e0cee3ac5505991beae26eb7e4c8b779e556248ada42d3748c6f2", "src/renigma/representation_fixed.py": "49bc04dc438b743f0b5e680519d7e2a145e3e4ce833c916fd60e2d94e5959b0e", "src/renigma/representation_fixed_diagnostics.py": "9e5dae675f2ef41fc59318134c7a3b2cb0cba44246b3d7defc449cf4977ee76b", "src/renigma/representation_fixed_study.py": "4ee7ba69ecb8c0c83f8d4d4b1ed7492242be63660d09d856cc92af977436b943", "src/renigma/representation_learning.py": "4abd7d319b643720ec36b08c9cbebb42ad10a030e786ff231912bc2f284dece3", "src/renigma/representation_math.py": "705208f79e6ebef7fff706423bb83e39b0d835a762d0be0ea29fa6db770b619b", "src/renigma/representation_study.py": "76b9d4b40bbf929f8d8deab9c79b57051f8552f143c55c7aaeefc23e68b12603", "src/renigma/resources.py": "003161b14e3afdba382ee3513f0624a7f07d5be3b94b12ac23e40928242dce50", "src/renigma/routing.py": "06e55cc7cf4bb97b66c83299fc84eea940d5713bd67c2a85ba20c155d389c4c0", "src/renigma/server.py": "9b703b2c40faaf651a4039d770e4914cf140dfd16100b57005a0eb83583164ec", "src/renigma/training.py": "b15ae472cae35d8f93205b69031af1105beeb7b5b2aeb459fc080f253d32ef36", "src/renigma/web/app.js": "6a1743905a01ca5870e53c885d74327a792e76b2df30235e9c810d24b9c36baa", "src/renigma/web/index.html": "07cae5696d05141cbee4da09cc3ad9a0d4c04080d8724d7a3a66b4a8d891f820", "src/renigma/web/styles.css": "4c1a818b3acece76076bb1754d59339d4275d3bb0fd7b67b4d433217332a19f9", "src/renigma/worker.py": "2b898ae8e22622d17936efe18613f470c89bfe02102f12afc0c8a85f8694cea2", "pyproject.toml": "e65d947f319af535facb7163870751a47cf48ee200f9ddaf96ece7e2f7230033", "uv.lock": "a23fa9ec75705d63d7d8d766489bcbcef99a6f2ed00b33d2e658a8005ce9bf86" } }, "script_sha256": "a54b50420af815cff97b78509a64f3064b6c8d93976c1e3005596c62ef3d697c", "helper_script_sha256": { "replay_representation.py": "1aee7e5d8a18b403f1dd72983c2233b2c92a016cea875b082ebc01495b797c5d", "replay_fixed_representation.py": "96f86b9caf60761e65bd341431055b94452c53ae989c757d7292b02bf5311bf6", "replay_head_intervention.py": "da6bdd54652127843838f02fcb12b4ff9660eb28eac26ebb31699591e7493dc9" }, "environment": { "python": "3.12.10", "numpy": "2.5.3" }, "current_source_required": true, "tolerances": { "continuous_atol": 1e-06, "continuous_rtol": 0.0001, "categorical": "exact decoded indices", "aggregate_accuracy_atol": 1e-07 }, "scope": "Twenty final H fitting checkpoints; NumPy inference, frozen-tensor isolation, independent truth, diagnostic/H-group and paired-correctness checks. Zero GPU or optimizer calls.", "limitations": [ "Same host, synthetic world and words; post-outcome recovery comparison, not independent-host or new-domain replication.", "Recorded training losses/gradients and optimizer state are contextual evidence, not independently replayed optimization.", "All continuous and categorical failures remain failures, even if task accuracy improves.", "Relation per-example CUDA arrays were not saved; relations compare recorded aggregate metrics." ], "trials": [ { "seed": 0, "arm": "native", "status": "passed", "files": { "seed-0-native/result.json": "17e6bbd872fffd4273596cc9ed4ca9f4f4d40a6894150d542c660ca712aab892", "seed-0-native/checkpoint.json": "e582fb862d817001b6c93efafcdccdc9daf7e52518b61b997694ca02a242cb57", "inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "seed-0-native/diagnostics-arrays.npz": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 3.8743019104003906e-07, "max_tolerance_ratio": 0.35905024250805767, "worst_tolerance_index": [ 64, 7 ], "actual_at_worst": 0.0007908046245574951, "saved_at_worst": 0.0007904171943664551, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 4.15187023463659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 4.151866596657783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6229368565750986e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444363291142508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 7.444350922014564e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.0001227772822666739 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926477193832397, "saved_at_worst": 1.2926477193832397, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4581003251396934, "worst_tolerance_index": [ 62, 8 ], "actual_at_worst": -0.0012227296829223633, "saved_at_worst": -0.0012222155928611755, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.00018554557755123824, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04766174405813217 }, "reported": { "count": 784, "observable_mse": 0.00018554567941464484, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04766174405813217 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.00010000780498627342 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 0.0002031843177974224, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05125661566853523 }, "reported": { "count": 580, "observable_mse": 0.00020318443421274424, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05125661566853523 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011409704742430952 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.00019697788229677826, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04766174778342247 }, "reported": { "count": 112, "observable_mse": 0.0001969779550563544, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04766174405813217 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006460592476540826 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 6.04273664066568e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "reported": { "count": 92, "observable_mse": 6.0427275457186624e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-11, "numeric_tolerance_ratio": 9.040318834088458e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.778126847464591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 98, "observable_mse": 3.7781326682306826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987823776900768 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003417623690158127 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.11751791741699e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0049878195859491825 }, "reported": { "count": 70, "observable_mse": 3.1175259209703654e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987821914255619 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.001553465506768622 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.356186400400475e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 14, "observable_mse": 4.356192584964447e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498782517388463 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004349702472956356 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987813998013735 }, "reported": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498783215880394 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.816079020500183e-08, "numeric_tolerance_ratio": 0.012117022670509476 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7222198443487287e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297571886330843 }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529756722971797 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0030440218389961885 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.335614383104257e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529757235199213 }, "reported": { "count": 70, "observable_mse": 3.3356191124767065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003652826317988257 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0065819373703562e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297572817653418 }, "reported": { "count": 14, "observable_mse": 3.0065733881201595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003957228511153945 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.370883056661114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052975681610405445 }, "reported": { "count": 14, "observable_mse": 6.370881601469591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297568626701832 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003044021561014145 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7371144571807235e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311058834195137 }, "reported": { "count": 98, "observable_mse": 3.737111546797678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311060231178999 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0009124017806934365 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.151369674014859e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311061628162861 }, "reported": { "count": 70, "observable_mse": 3.151365672238171e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311059299856424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0015206697269865119 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.960085814469494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311049986630678 }, "reported": { "count": 14, "observable_mse": 3.960098547395319e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311064422130585 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4435499906539917e-08, "numeric_tolerance_ratio": 0.009428149153154155 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.442864105338231e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311055574566126 }, "reported": { "count": 14, "observable_mse": 6.442856829380617e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311062093824148 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004257874458583739 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.720241511473432e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "reported": { "count": 98, "observable_mse": 3.720237145898864e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681416642266 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.12783413392026e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518278449774 }, "reported": { "count": 72, "observable_mse": 3.127835225313902e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518744111061 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00030094078470998165 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.02617733925581e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "reported": { "count": 14, "observable_mse": 4.0261500544147566e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681507163908 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.917762220837176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499614402651787 }, "reported": { "count": 12, "observable_mse": 6.917752762092277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499619524925947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0033047741282601615 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.71916830772534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498461425304413 }, "reported": { "count": 98, "observable_mse": 3.719156302395277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498470738530159 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.006009125611987981 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.126173032796942e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0054947249591350555 }, "reported": { "count": 72, "observable_mse": 3.126166484435089e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00549473287537694 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.916241884231567e-09, "numeric_tolerance_ratio": 0.005108988937015792 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.025795715278946e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498459096997976 }, "reported": { "count": 14, "observable_mse": 4.0257600630866364e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498471204191446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.210719347000122e-08, "numeric_tolerance_ratio": 0.007811863060872042 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.919406587257981e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055208331905305386 }, "reported": { "count": 12, "observable_mse": 6.919389124959707e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005520846229046583 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.30385160446167e-08, "numeric_tolerance_ratio": 0.008400647652971193 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.75562631234061e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 98, "observable_mse": 3.755628131330013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.164217559969984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217758007347584 }, "reported": { "count": 70, "observable_mse": 3.164222289342433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002141989845922839 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.49347566650249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217755679041147 }, "reported": { "count": 14, "observable_mse": 6.493463297374547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0036719825930105815 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.001238399650901556, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405950367450714 }, "reported": { "count": 98, "observable_mse": 0.0012384005822241306, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405951857566833 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0010343753291123703 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.0012248106068000197, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405951857566833 }, "reported": { "count": 84, "observable_mse": 0.0012248115381225944, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405951857566833 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0008296999655205318 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00131993368268013, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405951857566833 }, "reported": { "count": 14, "observable_mse": 0.001319935079663992, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13405951857566833 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0012340917612088322 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640167688019574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707476753741503 }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005929166943850281 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 2.1136473151273094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005575723946094513 }, "reported": { "count": 72, "observable_mse": 2.1136505893082358e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055757262744009495 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0014948300936472665 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.2392847313312814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707480479031801 }, "reported": { "count": 14, "observable_mse": 2.239291643491015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0017787500831550841 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.3784225049894303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441648583859205 }, "reported": { "count": 12, "observable_mse": 2.37841213674983e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441639736294746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.005381193482372771 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2945678234100342, "saved_at_worst": 1.2945678234100342, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5731212112389529, "worst_tolerance_index": [ 1729, 8 ], "actual_at_worst": -0.0003994107246398926, "saved_at_worst": -0.0004000067710876465, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.00066754239378497, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07171070575714111 }, "reported": { "count": 6272, "observable_mse": 0.0006675425102002919, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07171070575714111 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010913040347916929 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.0007636122754774988, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07652070373296738 }, "reported": { "count": 4092, "observable_mse": 0.0007636125083081424, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07652070373296738 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00021631273280615958 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.0006516751018352807, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07171070575714111 }, "reported": { "count": 896, "observable_mse": 0.0006516751600429416, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07171070575714111 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.464648521372365e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.00037244788836687803, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05357403680682182 }, "reported": { "count": 1284, "observable_mse": 0.0003724479756783694, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.05357403680682182 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.41763598862407e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.0944832563400269, "saved_at_worst": 1.0944832563400269, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.332993507385254e-07, "max_tolerance_ratio": 0.5058330479705814, "worst_tolerance_index": [ 1528, 4 ], "actual_at_worst": -0.0008991509675979614, "saved_at_worst": -0.0008997023105621338, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0017343565123155713, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10217385739088058 }, "reported": { "count": 3136, "observable_mse": 0.001734356745146215, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10217385739088058 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00019841790113478305 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.0018767246510833502, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10587301105260849 }, "reported": { "count": 1582, "observable_mse": 0.0018767250003293157, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10587301105260849 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00029405914969919786 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.001719222986139357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10217385739088058 }, "reported": { "count": 448, "observable_mse": 0.0017192234518006444, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10217385739088058 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00039734824514860753 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.0015368456952273846, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09663684666156769 }, "reported": { "count": 1106, "observable_mse": 0.0015368463937193155, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09663684666156769 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0006054444231327111 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.0646353960037231, "saved_at_worst": 1.0646353960037231, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4339908584165583, "worst_tolerance_index": [ 2639, 4 ], "actual_at_worst": 0.0016734600067138672, "saved_at_worst": 0.001673966646194458, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.009501377120614052, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.21004949510097504 }, "reported": { "count": 3136, "observable_mse": 0.009501378051936626, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.21004948019981384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0006771732057861167 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.012110286392271519, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.2318454086780548 }, "reported": { "count": 1001, "observable_mse": 0.012110287323594093, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.23184539377689362 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0006161440977285771 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.009507632814347744, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.21004949510097504 }, "reported": { "count": 448, "observable_mse": 0.009507635608315468, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.21004949510097504 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0014322431379923146 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.007951690815389156, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.19597406685352325 }, "reported": { "count": 1687, "observable_mse": 0.007951691746711731, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.19597406685352325 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005187937648194476 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.0119585990905762, "saved_at_worst": 1.0119585990905762, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.41252770307847064, "worst_tolerance_index": [ 1895, 0 ], "actual_at_worst": -0.000611158087849617, "saved_at_worst": -0.0006107203662395477, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.023222893476486206, "exact_state_accuracy": 0.9422831632653061, "per_field_accuracy": { "x": 0.984375, "y": 0.9824617346938775, "h": 0.9716198979591837 }, "normalized_latent_rmse": 0.30537670850753784 }, "reported": { "count": 3136, "observable_mse": 0.023222893476486206, "exact_state_accuracy": 0.9422831535339355, "per_field_accuracy": { "x": 0.984375, "y": 0.98246169090271, "h": 0.9716199040412903 }, "normalized_latent_rmse": 0.30537667870521545 }, "differences": [], "max_errors": { "accuracy_abs": 4.379116758101986e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0009449754658476739 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0301845520734787, "exact_state_accuracy": 0.9459459459459459, "per_field_accuracy": { "x": 1.0, "y": 0.9754299754299754, "h": 0.9705159705159705 }, "normalized_latent_rmse": 0.3464100658893585 }, "reported": { "count": 407, "observable_mse": 0.030184555798768997, "exact_state_accuracy": 0.9459459185600281, "per_field_accuracy": { "x": 1.0, "y": 0.9754299521446228, "h": 0.9705159664154053 }, "normalized_latent_rmse": 0.3464100658893585 }, "differences": [], "max_errors": { "accuracy_abs": 2.7385917866773468e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0009270452850385952 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.023366834968328476, "exact_state_accuracy": 0.9375, "per_field_accuracy": { "x": 0.984375, "y": 0.984375, "h": 0.9642857142857143 }, "normalized_latent_rmse": 0.30537667870521545 }, "reported": { "count": 448, "observable_mse": 0.023366834968328476, "exact_state_accuracy": 0.9375000596046448, "per_field_accuracy": { "x": 0.9843750596046448, "y": 0.9843750596046448, "h": 0.9642857313156128 }, "normalized_latent_rmse": 0.30537667870521545 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.021952450275421143, "exact_state_accuracy": 0.9425690486628672, "per_field_accuracy": { "x": 0.981587023235423, "y": 0.9833406400701447, "h": 0.9732573432704954 }, "normalized_latent_rmse": 0.2974605858325958 }, "reported": { "count": 2281, "observable_mse": 0.021952450275421143, "exact_state_accuracy": 0.9425690770149231, "per_field_accuracy": { "x": 0.9815870523452759, "y": 0.9833406209945679, "h": 0.9732573628425598 }, "normalized_latent_rmse": 0.2974605858325958 }, "differences": [], "max_errors": { "accuracy_abs": 2.9109852861530783e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128660127520561 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.758500679396093e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7585035897791386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.899485319690801e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128450579941273 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7913549022050574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7913512642262504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6242380475881334e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010601093992590904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.960090907639824e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.960088361054659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.536540241476581e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010598846711218357 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.98074698750861e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.980756446253508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.42124123167376e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03354682773351669 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00016841452452354133, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001684145099716261, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4310898925389558e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.034877024590969086 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002028434246312827, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00020284306083340198, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-10, "numeric_tolerance_ratio": 0.00035656520299299306 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.033779799938201904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018723947869148105, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018723943503573537, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.285336176055767e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020691297948360443 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.837881275918335e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.837849989300594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.00031043304973434846 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.23123303055763245 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.009352692402899265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00935269147157669, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004812367189252784 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.23123227059841156 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.011121547780930996, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.01112155057489872, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.00132280426758386 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0048739295452833176 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.30572659347672e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.3057307771523483e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.1740513894015114e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640156774083152e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.2671108366151636e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010431777685880661 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.1628161600092426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.162837260286324e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1100277081131935e-10, "numeric_tolerance_ratio": 0.0002099189932115251 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.72222057194449e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0873462981808747e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.857536648865789e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.8575337384827435e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8991992866106846e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7742513086413965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.774241849896498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.423179539860027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0025512666907161474, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.002551267622038722, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.000742014753139494 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.001361202565021813, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0013612019829452038, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005123371717257159 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 4.222623465466313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 4.222628194838762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.709486040680649e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 3.188768096151762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 3.1887459044810385e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.219167072325945e-10, "numeric_tolerance_ratio": 0.0002212113205401971 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1726252927910537e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.172617289237678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.978241468780203e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 5.0208916945848614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.020887329010293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.343765013792109e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 979, "error_count": 1, "per_field_correct_counts": { "x": 980, "y": 979, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 489, "error_count": 1, "per_field_correct_counts": { "x": 490, "y": 490, "h": 489 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 77, "error_count": 21, "per_field_correct_counts": { "x": 98, "y": 98, "h": 77 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 144, "error_count": 52, "per_field_correct_counts": { "x": 147, "y": 193, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 42, "error_count": 56, "per_field_correct_counts": { "x": 98, "y": 72, "h": 63 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 48, "error_count": 50, "per_field_correct_counts": { "x": 98, "y": 73, "h": 66 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 2955, "accuracy": 0.9422831632653061 }, "numpy": { "cases": 3136, "correct": 2955, "accuracy": 0.9422831632653061 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0012248115381225944, "latent_mse": 0.017971953377127647, "scaled_prediction": 0.00017738650785759091, "scaled_weighted_latent": 0.00026028347201645374, "total": 0.00043766997987404466 }, "gradients": { "norms": { "prediction": 0.0018048041965812445, "weighted_latent": 0.0014241763856261969, "total": 0.0028967412654310465 }, "prediction_weighted_latent_dot": 1.5527552932326216e-06, "prediction_weighted_latent_cosine": 0.6041004313564027, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "e582fb862d817001b6c93efafcdccdc9daf7e52518b61b997694ca02a242cb57" } ], "final_losses": { "prediction_mse": 0.0012248115381225944, "latent_mse": 0.017971953377127647, "scaled_prediction": 0.00017738650785759091, "scaled_weighted_latent": 0.00026028347201645374, "total": 0.00043766997987404466 }, "training_seconds": 0.0001259999698959291, "evaluation_seconds": 0.7381863999762572 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 0, "arm": "prediction_latent", "status": "passed", "files": { "seed-0-prediction_latent/result.json": "b96bd2283b5fe3039b095f345c3a317a648d7e275d5b5d23e25ff4556f5d74e6", "seed-0-prediction_latent/checkpoint.json": "5841c52b66285b7a9dbfc26dcf897f5a2dfb493fc8a30c68e2de0daafc205dee", "inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "seed-0-prediction_latent/diagnostics-arrays.npz": "4c9966db1686750b758d3fd35adc6821f0983f63b697220ab5ce91c45b8c783d" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 3.8743019104003906e-07, "max_tolerance_ratio": 0.35905024250805767, "worst_tolerance_index": [ 64, 7 ], "actual_at_worst": 0.0007908046245574951, "saved_at_worst": 0.0007904171943664551, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 4.15187023463659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 4.151866596657783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6229368565750986e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444363291142508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 7.444350922014564e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.0001227772822666739 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926477193832397, "saved_at_worst": 1.2926477193832397, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 6.705522537231445e-07, "max_tolerance_ratio": 0.4581003251396934, "worst_tolerance_index": [ 62, 8 ], "actual_at_worst": -0.0012227296829223633, "saved_at_worst": -0.0012222155928611755, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 3.544173887348734e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005298294126987457 }, "reported": { "count": 784, "observable_mse": 3.5441717045614496e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005298294126987457 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1750784328182317e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 3.060372546315193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005271988920867443 }, "reported": { "count": 580, "observable_mse": 3.060372546315193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005271988920867443 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 3.9971819205675274e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529829366132617 }, "reported": { "count": 112, "observable_mse": 3.997175008407794e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529829366132617 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.912159733474255e-11, "numeric_tolerance_ratio": 6.884640620045939e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 6.04273664066568e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "reported": { "count": 92, "observable_mse": 6.0427275457186624e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-11, "numeric_tolerance_ratio": 9.040318834088458e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.778126847464591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 98, "observable_mse": 3.7781326682306826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987823776900768 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003417623690158127 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.11751791741699e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0049878195859491825 }, "reported": { "count": 70, "observable_mse": 3.1175259209703654e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987821914255619 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.001553465506768622 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.356186400400475e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 14, "observable_mse": 4.356192584964447e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498782517388463 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004349702472956356 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987813998013735 }, "reported": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498783215880394 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.816079020500183e-08, "numeric_tolerance_ratio": 0.012117022670509476 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7222198443487287e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297571886330843 }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529756722971797 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0030440218389961885 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.335614383104257e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529757235199213 }, "reported": { "count": 70, "observable_mse": 3.3356191124767065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003652826317988257 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0065819373703562e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297572817653418 }, "reported": { "count": 14, "observable_mse": 3.0065733881201595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003957228511153945 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.370883056661114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052975681610405445 }, "reported": { "count": 14, "observable_mse": 6.370881601469591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297568626701832 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003044021561014145 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7371144571807235e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311058834195137 }, "reported": { "count": 98, "observable_mse": 3.737111546797678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311060231178999 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0009124017806934365 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.151369674014859e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311061628162861 }, "reported": { "count": 70, "observable_mse": 3.151365672238171e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311059299856424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0015206697269865119 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.960085814469494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311049986630678 }, "reported": { "count": 14, "observable_mse": 3.960098547395319e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311064422130585 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4435499906539917e-08, "numeric_tolerance_ratio": 0.009428149153154155 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.442864105338231e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311055574566126 }, "reported": { "count": 14, "observable_mse": 6.442856829380617e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311062093824148 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004257874458583739 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.720241511473432e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "reported": { "count": 98, "observable_mse": 3.720237145898864e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681416642266 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.12783413392026e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518278449774 }, "reported": { "count": 72, "observable_mse": 3.127835225313902e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518744111061 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00030094078470998165 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.02617733925581e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "reported": { "count": 14, "observable_mse": 4.0261500544147566e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681507163908 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.917762220837176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499614402651787 }, "reported": { "count": 12, "observable_mse": 6.917752762092277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499619524925947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0033047741282601615 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.71916830772534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498461425304413 }, "reported": { "count": 98, "observable_mse": 3.719156302395277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498470738530159 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.006009125611987981 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.126173032796942e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0054947249591350555 }, "reported": { "count": 72, "observable_mse": 3.126166484435089e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00549473287537694 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.916241884231567e-09, "numeric_tolerance_ratio": 0.005108988937015792 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.025795715278946e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498459096997976 }, "reported": { "count": 14, "observable_mse": 4.0257600630866364e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498471204191446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.210719347000122e-08, "numeric_tolerance_ratio": 0.007811863060872042 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.919406587257981e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055208331905305386 }, "reported": { "count": 12, "observable_mse": 6.919389124959707e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005520846229046583 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.30385160446167e-08, "numeric_tolerance_ratio": 0.008400647652971193 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.75562631234061e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 98, "observable_mse": 3.755628131330013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.164217559969984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217758007347584 }, "reported": { "count": 70, "observable_mse": 3.164222289342433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002141989845922839 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.49347566650249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217755679041147 }, "reported": { "count": 14, "observable_mse": 6.493463297374547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0036719825930105815 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.756877777050249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004836500622332096 }, "reported": { "count": 98, "observable_mse": 3.756875958060846e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0048364996910095215 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006277239200697941 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 3.3182703191414475e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0048364996910095215 }, "reported": { "count": 84, "observable_mse": 3.318267044960521e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004836499225348234 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003138619698858303 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 6.388524343492463e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004836504813283682 }, "reported": { "count": 14, "observable_mse": 6.388530164258555e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004836502484977245 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.001569309504646551 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640167688019574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707476753741503 }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005929166943850281 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 2.1136473151273094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005575723946094513 }, "reported": { "count": 72, "observable_mse": 2.1136505893082358e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055757262744009495 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0014948300936472665 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.2392847313312814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707480479031801 }, "reported": { "count": 14, "observable_mse": 2.239291643491015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0017787500831550841 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.3784225049894303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441648583859205 }, "reported": { "count": 12, "observable_mse": 2.37841213674983e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441639736294746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.005381193482372771 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2945678234100342, "saved_at_worst": 1.2945678234100342, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5731212112389529, "worst_tolerance_index": [ 1729, 8 ], "actual_at_worst": -0.0003994107246398926, "saved_at_worst": -0.0004000067710876465, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 3.810602720477618e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010186481289565563 }, "reported": { "count": 6272, "observable_mse": 3.810604175669141e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010186481289565563 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4496674141350507e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 3.450327494647354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010137348435819149 }, "reported": { "count": 4092, "observable_mse": 3.450328949838877e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010137349367141724 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00046248518493457986 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 4.079767313669436e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010186480358242989 }, "reported": { "count": 896, "observable_mse": 4.07976949645672e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010186480358242989 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1739181990985534e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 4.770941086462699e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010341506451368332 }, "reported": { "count": 1284, "observable_mse": 4.770942541654222e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.010341506451368332 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4482818533303257e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.276785135269165, "saved_at_worst": 1.276785135269165, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.332993507385254e-07, "max_tolerance_ratio": 0.5058330479705814, "worst_tolerance_index": [ 1528, 4 ], "actual_at_worst": -0.0008991509675979614, "saved_at_worst": -0.0008997023105621338, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 9.610591223463416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019914837554097176 }, "reported": { "count": 3136, "observable_mse": 9.610602864995599e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019914837554097176 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011530715059507135 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 9.450990910409018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019731320440769196 }, "reported": { "count": 1582, "observable_mse": 9.451002551941201e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019731322303414345 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006264925354554616 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 9.646607941249385e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019914837554097176 }, "reported": { "count": 448, "observable_mse": 9.646618855185807e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.019914837554097176 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-10, "numeric_tolerance_ratio": 0.00010809659753677173 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 9.824293374549598e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020174432545900345 }, "reported": { "count": 1106, "observable_mse": 9.824305743677542e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020174432545900345 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.00012248792065866024 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.279247760772705, "saved_at_worst": 1.279247760772705, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4339908584165583, "worst_tolerance_index": [ 2639, 4 ], "actual_at_worst": 0.0016734600067138672, "saved_at_worst": 0.001673966646194458, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0004068507405463606, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.038872551172971725 }, "reported": { "count": 3136, "observable_mse": 0.00040685111889615655, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03887254744768143 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0007622459832792379 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.0004357768630143255, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03872304782271385 }, "reported": { "count": 1001, "observable_mse": 0.00043577715405263007, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03872304782271385 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002788851278357503 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00040604922105558217, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.038872551172971725 }, "reported": { "count": 448, "observable_mse": 0.0004060495411977172, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03887254744768143 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0007622459832792379 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.0003898999420925975, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03896098956465721 }, "reported": { "count": 1687, "observable_mse": 0.0003899003495462239, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03896098956465721 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.0745362639427185e-10, "numeric_tolerance_ratio": 0.0003921631706622357 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2058404684066772, "saved_at_worst": 1.2058404684066772, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.811452865600586e-07, "max_tolerance_ratio": 0.49983340846515517, "worst_tolerance_index": [ 2358, 8 ], "actual_at_worst": 0.0001366734504699707, "saved_at_worst": 0.00013616681098937988, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0017277415608987212, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0770382285118103 }, "reported": { "count": 3136, "observable_mse": 0.0017277424922212958, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0770382210612297 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0008560125087669803 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0017671252135187387, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07580389082431793 }, "reported": { "count": 407, "observable_mse": 0.0017671257955953479, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07580388337373734 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0008683267358041612 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0017325704684481025, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0770382210612297 }, "reported": { "count": 448, "observable_mse": 0.0017325716326013207, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0770382210612297 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0009922404522419563 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.001719765947200358, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07725640386343002 }, "reported": { "count": 2281, "observable_mse": 0.0017197668785229325, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07725639641284943 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.000853872140406964 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128660127520561 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.758500679396093e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7585035897791386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.899485319690801e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128450579941273 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7913549022050574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7913512642262504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6242380475881334e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010601093992590904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.960090907639824e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.960088361054659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.536540241476581e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010598846711218357 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.98074698750861e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.980756446253508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.42124123167376e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03354682773351669 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00016841452452354133, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001684145099716261, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4310898925389558e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.034877024590969086 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002028434246312827, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00020284306083340198, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-10, "numeric_tolerance_ratio": 0.00035656520299299306 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.033779799938201904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018723947869148105, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018723943503573537, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.285336176055767e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020691297948360443 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.837881275918335e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.837849989300594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.00031043304973434846 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.009375471621751785 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.292930316296406e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.292925586923957e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.709156391254385e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01870586909353733 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.81131854839623e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.81130326888524e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.5279510989785194e-10, "numeric_tolerance_ratio": 0.0001514605451016903 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0048739295452833176 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.30572659347672e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.3057307771523483e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.1740513894015114e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640156774083152e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.2671108366151636e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010431777685880661 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.1628161600092426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.162837260286324e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1100277081131935e-10, "numeric_tolerance_ratio": 0.0002099189932115251 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.72222057194449e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0873462981808747e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.857536648865789e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.8575337384827435e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8991992866106846e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7742513086413965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.774241849896498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.423179539860027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.8412308640545234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.8412283174693584e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5368405810872216e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.37335365859326e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.3733540223911405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6221379156685686e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 4.222623465466313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 4.222628194838762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.709486040680649e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 3.188768096151762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 3.1887459044810385e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.219167072325945e-10, "numeric_tolerance_ratio": 0.0002212113205401971 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1726252927910537e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.172617289237678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.978241468780203e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 5.0208916945848614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.020887329010293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.343765013792109e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0012248115381225944, "latent_mse": 0.017971953377127647, "scaled_prediction": 0.00017738650785759091, "scaled_weighted_latent": 0.00026028347201645374, "total": 0.00043766997987404466 }, "gradients": { "norms": { "prediction": 0.0018048041965812445, "weighted_latent": 0.0014241763856261969, "total": 0.0028967412654310465 }, "prediction_weighted_latent_dot": 1.5527552932326216e-06, "prediction_weighted_latent_cosine": 0.6041004313564027, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0012890921207144856, "latent_mse": 0.016826288774609566, "scaled_prediction": 0.00018669611017685384, "scaled_weighted_latent": 0.0002436910872347653, "total": 0.00043038721196353436 }, "gradients": { "norms": { "prediction": 0.0021970062516629696, "weighted_latent": 0.0013744073221459985, "total": 0.003139321692287922 }, "prediction_weighted_latent_dot": 1.5697547723902971e-06, "prediction_weighted_latent_cosine": 0.5198583920528568, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.030365290120244026 }, { "step": 100, "losses": { "prediction_mse": 3.1077976018423215e-05, "latent_mse": 0.00010193680645897985, "scaled_prediction": 4.500948307395447e-06, "scaled_weighted_latent": 1.4763262470296468e-06, "total": 5.9772746681119315e-06 }, "gradients": { "norms": { "prediction": 7.528421701863408e-05, "weighted_latent": 0.00011191774683538824, "total": 0.0001511855807621032 }, "prediction_weighted_latent_dot": 2.3318922348636306e-09, "prediction_weighted_latent_cosine": 0.27676143903546707, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0008034161874093115 }, { "step": 200, "losses": { "prediction_mse": 3.287775689386763e-05, "latent_mse": 2.3269645680557005e-05, "scaled_prediction": 4.761606305692112e-06, "scaled_weighted_latent": 3.3700865742503083e-07, "total": 5.098615019960562e-06 }, "gradients": { "norms": { "prediction": 5.176300328457728e-05, "weighted_latent": 5.117325781611726e-05, "total": 6.790669431211427e-07 }, "prediction_weighted_latent_dot": -2.64882471512351e-09, "prediction_weighted_latent_cosine": -0.9999785580085703, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.1257843551866245e-06 }, { "step": 300, "losses": { "prediction_mse": 3.285860293544829e-05, "latent_mse": 2.3463931938749738e-05, "scaled_prediction": 4.758832346851705e-06, "scaled_weighted_latent": 3.398224635020597e-07, "total": 5.098654582980089e-06 }, "gradients": { "norms": { "prediction": 5.168217103346251e-05, "weighted_latent": 5.139409040566534e-05, "total": 3.7041045288788155e-06 }, "prediction_weighted_latent_dot": -2.649339858606936e-09, "prediction_weighted_latent_cosine": -0.9974330173843801, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.452952973428182e-05 }, { "step": 400, "losses": { "prediction_mse": 3.285737329861149e-05, "latent_mse": 2.34740182349924e-05, "scaled_prediction": 4.758654085890157e-06, "scaled_weighted_latent": 3.3996855108853197e-07, "total": 5.098622750665527e-06 }, "gradients": { "norms": { "prediction": 5.1515187806217e-05, "weighted_latent": 5.141399378771894e-05, "total": 1.8833698050002567e-06 }, "prediction_weighted_latent_dot": -2.6468331970619374e-09, "prediction_weighted_latent_cosine": -0.9993323462402814, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.012595672975294e-05 }, { "step": 500, "losses": { "prediction_mse": 3.286363062215969e-05, "latent_mse": 2.3442897145287134e-05, "scaled_prediction": 4.759560397360474e-06, "scaled_weighted_latent": 3.395178396203846e-07, "total": 5.099078407511115e-06 }, "gradients": { "norms": { "prediction": 5.302550198393874e-05, "weighted_latent": 5.134254024596885e-05, "total": 1.133940622821683e-05 }, "prediction_weighted_latent_dot": -2.6595889934810657e-09, "prediction_weighted_latent_cosine": -0.9769051209153155, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00024398861569352448 }, { "step": 600, "losses": { "prediction_mse": 3.291038956376724e-05, "latent_mse": 2.338582999072969e-05, "scaled_prediction": 4.766332494909875e-06, "scaled_weighted_latent": 3.3869133631014847e-07, "total": 5.105023774376605e-06 }, "gradients": { "norms": { "prediction": 6.272921746131033e-05, "weighted_latent": 5.1213362894486636e-05, "total": 3.5458531783660874e-05 }, "prediction_weighted_latent_dot": -2.6502278149820313e-09, "prediction_weighted_latent_cosine": -0.8249545903674677, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0006560756010003388 }, { "step": 700, "losses": { "prediction_mse": 3.305563950561918e-05, "latent_mse": 2.3388383851852268e-05, "scaled_prediction": 4.787368652614532e-06, "scaled_weighted_latent": 3.3872831295411743e-07, "total": 5.1260967666166835e-06 }, "gradients": { "norms": { "prediction": 8.387726120417938e-05, "weighted_latent": 5.115637759445235e-05, "total": 6.733044574502856e-05 }, "prediction_weighted_latent_dot": -2.559490397402442e-09, "prediction_weighted_latent_cosine": -0.5964986690100962, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0019195745699107647 }, { "step": 800, "losses": { "prediction_mse": 3.2857918995432556e-05, "latent_mse": 2.3521432012785226e-05, "scaled_prediction": 4.758733211929211e-06, "scaled_weighted_latent": 3.4065521958837053e-07, "total": 5.09938854520442e-06 }, "gradients": { "norms": { "prediction": 5.2239465730963275e-05, "weighted_latent": 5.15259598614648e-05, "total": 1.2796896044164896e-05 }, "prediction_weighted_latent_dot": -2.610063054575562e-09, "prediction_weighted_latent_cosine": -0.9696749618716722, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.000438886258052662 }, { "step": 900, "losses": { "prediction_mse": 3.3233754948014393e-05, "latent_mse": 2.3964656065800227e-05, "scaled_prediction": 4.813164650840918e-06, "scaled_weighted_latent": 3.4707434792835556e-07, "total": 5.160239197721239e-06 }, "gradients": { "norms": { "prediction": 0.00012345743016339839, "weighted_latent": 5.250451431493275e-05, "total": 0.00012060565495630726 }, "prediction_weighted_latent_dot": -1.726368381582688e-09, "prediction_weighted_latent_cosine": -0.26632969713607313, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0023534814827144146 }, { "step": 1000, "losses": { "prediction_mse": 3.318267044960521e-05, "latent_mse": 2.3391723516397178e-05, "scaled_prediction": 4.805766366189346e-06, "scaled_weighted_latent": 3.38776686703568e-07, "total": 5.144543138158042e-06 }, "gradients": { "norms": { "prediction": 0.0001157012811745517, "weighted_latent": 5.10505742568057e-05, "total": 0.0001058107809512876 }, "prediction_weighted_latent_dot": -2.3985131658577075e-09, "prediction_weighted_latent_cosine": -0.40607224546774595, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.001952397171407938 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "d49500456491e7766dc4885ab7c8a8b507ac8c95ca6f98d055d738e160c882af" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "7f5b766ed5121f19061035d8f83fd59f7b0f5632a11831bbf58a0ca2e3ef6e78" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "48dcbc552dba84b083d15627725a42753ad419ceae8d9f94dd08233d81d77f76" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "fd3c2fa5de0382152fcd0585bc269f8d60009f8fd0fd0e511b8107f09d9bdd78" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "d9e7afc97416c000c1dbe7ffd2789cdd9bf0e9762838172802cd6fcb5d84af3c" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "9212ac911a1e8a2941591140ad59df931d14a110fd616647edbc1d802e1be7c3" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "f645318aeed46a299612a4e849ecae3ba417d5bb05a118a6fda9fdfce1f0f131" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "d8ce3b1b087be62d94cc2c4188c1dc719b13cfbd1dfe9e1a1f6067d3b9a0257c" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "dbc77a4be9a4ce76a0d5b0519d01e22dfc900db42d6feae64113d06899cf38ae" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "bcdd748c39e446890935b2e68525714bb29fabfd5125318e83f42d8d1e373ad4" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "cb11b53b84855d552d86b9283b6b27b796cebf7d61ac27d0a15396ed9bfe5b52" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "5841c52b66285b7a9dbfc26dcf897f5a2dfb493fc8a30c68e2de0daafc205dee" } ], "final_losses": { "prediction_mse": 3.318267044960521e-05, "latent_mse": 2.3391723516397178e-05, "scaled_prediction": 4.805766366189346e-06, "scaled_weighted_latent": 3.38776686703568e-07, "total": 5.144543138158042e-06 }, "training_seconds": 3.7349936999962665, "evaluation_seconds": 0.5165338999941014 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 0, "arm": "latent_only", "status": "passed", "files": { "seed-0-latent_only/result.json": "3f52ebb15056db2e1e07bf859380250e0b771714ce9588fc4f64d80b36c7bb9b", "seed-0-latent_only/checkpoint.json": "d72fb76a891400e98d3b3de0e8d37bce8f8b9352e6c9f04e76f7a9779a002ce8", "inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "seed-0-latent_only/diagnostics-arrays.npz": "3da23297be049aeac4fd20279ba3652e82a286efbfe76a421e116b7d4b52ce34" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 3.8743019104003906e-07, "max_tolerance_ratio": 0.35905024250805767, "worst_tolerance_index": [ 64, 7 ], "actual_at_worst": 0.0007908046245574951, "saved_at_worst": 0.0007904171943664551, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 4.15187023463659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 4.151866596657783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6229368565750986e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444363291142508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 7.444350922014564e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.0001227772822666739 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926477193832397, "saved_at_worst": 1.2926477193832397, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 7.078051567077637e-07, "max_tolerance_ratio": 0.4581003251396934, "worst_tolerance_index": [ 62, 8 ], "actual_at_worst": -0.0012227296829223633, "saved_at_worst": -0.0012222155928611755, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 3.652337909443304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014773458242416 }, "reported": { "count": 784, "observable_mse": 3.65233572665602e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014773458242416 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1748440237272644e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 3.181095598847605e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004940252751111984 }, "reported": { "count": 580, "observable_mse": 3.1810952350497246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004940252751111984 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.626442747347944e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 4.129156150156632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014772526919842 }, "reported": { "count": 112, "observable_mse": 4.129150329390541e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014772526919842 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.79683010839524e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 6.04273664066568e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "reported": { "count": 92, "observable_mse": 6.0427275457186624e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-11, "numeric_tolerance_ratio": 9.040318834088458e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.778126847464591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 98, "observable_mse": 3.7781326682306826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987823776900768 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003417623690158127 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.11751791741699e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0049878195859491825 }, "reported": { "count": 70, "observable_mse": 3.1175259209703654e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987821914255619 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.001553465506768622 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.356186400400475e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 14, "observable_mse": 4.356192584964447e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498782517388463 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004349702472956356 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987813998013735 }, "reported": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498783215880394 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.816079020500183e-08, "numeric_tolerance_ratio": 0.012117022670509476 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7222198443487287e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297571886330843 }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529756722971797 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0030440218389961885 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.335614383104257e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529757235199213 }, "reported": { "count": 70, "observable_mse": 3.3356191124767065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003652826317988257 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0065819373703562e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297572817653418 }, "reported": { "count": 14, "observable_mse": 3.0065733881201595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003957228511153945 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.370883056661114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052975681610405445 }, "reported": { "count": 14, "observable_mse": 6.370881601469591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297568626701832 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003044021561014145 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7371144571807235e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311058834195137 }, "reported": { "count": 98, "observable_mse": 3.737111546797678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311060231178999 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0009124017806934365 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.151369674014859e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311061628162861 }, "reported": { "count": 70, "observable_mse": 3.151365672238171e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311059299856424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0015206697269865119 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.960085814469494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311049986630678 }, "reported": { "count": 14, "observable_mse": 3.960098547395319e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311064422130585 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4435499906539917e-08, "numeric_tolerance_ratio": 0.009428149153154155 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.442864105338231e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311055574566126 }, "reported": { "count": 14, "observable_mse": 6.442856829380617e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311062093824148 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004257874458583739 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.720241511473432e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "reported": { "count": 98, "observable_mse": 3.720237145898864e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681416642266 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.12783413392026e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518278449774 }, "reported": { "count": 72, "observable_mse": 3.127835225313902e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518744111061 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00030094078470998165 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.02617733925581e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "reported": { "count": 14, "observable_mse": 4.0261500544147566e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681507163908 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.917762220837176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499614402651787 }, "reported": { "count": 12, "observable_mse": 6.917752762092277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499619524925947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0033047741282601615 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.71916830772534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498461425304413 }, "reported": { "count": 98, "observable_mse": 3.719156302395277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498470738530159 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.006009125611987981 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.126173032796942e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0054947249591350555 }, "reported": { "count": 72, "observable_mse": 3.126166484435089e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00549473287537694 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.916241884231567e-09, "numeric_tolerance_ratio": 0.005108988937015792 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.025795715278946e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498459096997976 }, "reported": { "count": 14, "observable_mse": 4.0257600630866364e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498471204191446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.210719347000122e-08, "numeric_tolerance_ratio": 0.007811863060872042 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.919406587257981e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055208331905305386 }, "reported": { "count": 12, "observable_mse": 6.919389124959707e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005520846229046583 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.30385160446167e-08, "numeric_tolerance_ratio": 0.008400647652971193 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.75562631234061e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 98, "observable_mse": 3.755628131330013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.164217559969984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217758007347584 }, "reported": { "count": 70, "observable_mse": 3.164222289342433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002141989845922839 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.49347566650249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217755679041147 }, "reported": { "count": 14, "observable_mse": 6.493463297374547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0036719825930105815 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622189226211049e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.541295091556094e-07 }, "reported": { "count": 98, "observable_mse": 4.6221881348174065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5176362921920372e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3658799364056904e-09, "numeric_tolerance_ratio": 0.002365844031498052 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 4.1518342186464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5466694947008364e-07 }, "reported": { "count": 84, "observable_mse": 4.151833854848519e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5143542952955613e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2315199405275052e-09, "numeric_tolerance_ratio": 0.0032314710046075458 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444318907801062e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5086457949564647e-07 }, "reported": { "count": 14, "observable_mse": 7.444314542226493e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5371813333331374e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.8535538376672775e-09, "numeric_tolerance_ratio": 0.002853509974044612 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640167688019574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707476753741503 }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005929166943850281 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 2.1136473151273094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005575723946094513 }, "reported": { "count": 72, "observable_mse": 2.1136505893082358e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055757262744009495 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0014948300936472665 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.2392847313312814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707480479031801 }, "reported": { "count": 14, "observable_mse": 2.239291643491015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0017787500831550841 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.3784225049894303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441648583859205 }, "reported": { "count": 12, "observable_mse": 2.37841213674983e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441639736294746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.005381193482372771 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2945678234100342, "saved_at_worst": 1.2945678234100342, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5731212112389529, "worst_tolerance_index": [ 1729, 8 ], "actual_at_worst": -0.0003994107246398926, "saved_at_worst": -0.0004000067710876465, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 3.762717096833512e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009358827956020832 }, "reported": { "count": 6272, "observable_mse": 3.7627178244292736e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009358827024698257 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00048108419659274 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 3.382581053301692e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009196789935231209 }, "reported": { "count": 4092, "observable_mse": 3.382582144695334e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009196789935231209 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0877143589583724e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 4.0451624954584986e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009358827024698257 }, "reported": { "count": 896, "observable_mse": 4.045164678245783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009358827024698257 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1739931240588358e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 4.777082722284831e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009857474826276302 }, "reported": { "count": 1284, "observable_mse": 4.777083086082712e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.009857474826276302 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6206825059325476e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2796679735183716, "saved_at_worst": 1.2796679735183716, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.332993507385254e-07, "max_tolerance_ratio": 0.5058330479705814, "worst_tolerance_index": [ 1528, 4 ], "actual_at_worst": -0.0008991509675979614, "saved_at_worst": -0.0008997023105621338, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 8.320776396431029e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01805855706334114 }, "reported": { "count": 3136, "observable_mse": 8.320785127580166e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01805855706334114 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.65909863785599e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 8.007911674212664e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01770593412220478 }, "reported": { "count": 1582, "observable_mse": 8.007918222574517e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01770593412220478 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.548361852765083e-11, "numeric_tolerance_ratio": 6.496339695735568e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 8.366630208911374e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01805855520069599 }, "reported": { "count": 448, "observable_mse": 8.366639667656273e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01805855520069599 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.380263612802507e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 8.749716653255746e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0185512974858284 }, "reported": { "count": 1106, "observable_mse": 8.74972902238369e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0185512974858284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.00012261840165349238 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2795177698135376, "saved_at_worst": 1.2795177698135376, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4339908584165583, "worst_tolerance_index": [ 2639, 4 ], "actual_at_worst": 0.0016734600067138672, "saved_at_worst": 0.001673966646194458, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00030474868253804743, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.033686716109514236 }, "reported": { "count": 3136, "observable_mse": 0.000304748973576352, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.033686716109514236 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002824312414728621 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00030883034924045205, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03248004615306854 }, "reported": { "count": 1001, "observable_mse": 0.0003088305820710957, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03248004615306854 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00022585553404942346 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.000303991895634681, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.033686719834804535 }, "reported": { "count": 448, "observable_mse": 0.0003039921575691551, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.033686719834804535 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.6193447411060333e-10, "numeric_tolerance_ratio": 0.00025420678714141903 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00030252774013206363, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.034382693469524384 }, "reported": { "count": 1687, "observable_mse": 0.0003025280893780291, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.034382693469524384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.0003389905491651891 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2083498239517212, "saved_at_worst": 1.2083498239517212, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.364418029785156e-07, "max_tolerance_ratio": 0.40262168358783795, "worst_tolerance_index": [ 2786, 5 ], "actual_at_worst": 0.0007334277033805847, "saved_at_worst": 0.0007329955697059631, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0013606043066829443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06840331107378006 }, "reported": { "count": 3136, "observable_mse": 0.0013606048887595534, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06840331852436066 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0009502889338298688 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0012911871308460832, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06452718377113342 }, "reported": { "count": 407, "observable_mse": 0.0012911877129226923, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06452718377113342 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005155140663089774 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.001364032388664782, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06840331107378006 }, "reported": { "count": 448, "observable_mse": 0.001364033087156713, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06840331107378006 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0006146514407380804 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0013723173178732395, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06907206773757935 }, "reported": { "count": 2281, "observable_mse": 0.0013723178999498487, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06907206773757935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005118363857356124 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128660127520561 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.758500679396093e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7585035897791386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.899485319690801e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128450579941273 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7913549022050574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7913512642262504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6242380475881334e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010601093992590904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.960090907639824e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.960088361054659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.536540241476581e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010598846711218357 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.98074698750861e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.980756446253508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.42124123167376e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03354682773351669 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00016841452452354133, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001684145099716261, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4310898925389558e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.034877024590969086 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002028434246312827, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00020284306083340198, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-10, "numeric_tolerance_ratio": 0.00035656520299299306 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.033779799938201904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018723947869148105, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018723943503573537, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.285336176055767e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020691297948360443 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.837881275918335e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.837849989300594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.00031043304973434846 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 2.529685048102692e-07 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.6221557568060234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622161577572115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.793985354858499e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00969147402793169 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.399236058816314e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.3992276914650574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.36735125631094e-11, "numeric_tolerance_ratio": 8.330702598749163e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0048739295452833176 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.30572659347672e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.3057307771523483e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.1740513894015114e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640156774083152e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.2671108366151636e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010431777685880661 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.1628161600092426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.162837260286324e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1100277081131935e-10, "numeric_tolerance_ratio": 0.0002099189932115251 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.72222057194449e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0873462981808747e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.857536648865789e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.8575337384827435e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8991992866106846e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7742513086413965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.774241849896498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.423179539860027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.755614307010546e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7556128518190235e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4497468349913079e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.206630910630338e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.2066290916409343e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8113696433085992e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 4.222623465466313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 4.222628194838762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.709486040680649e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 3.188768096151762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 3.1887459044810385e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.219167072325945e-10, "numeric_tolerance_ratio": 0.0002212113205401971 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1726252927910537e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.172617289237678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.978241468780203e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 5.0208916945848614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.020887329010293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.343765013792109e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0012248115381225944, "latent_mse": 0.017971953377127647, "scaled_prediction": 0.00017738650785759091, "scaled_weighted_latent": 0.00026028347201645374, "total": 0.00026028347201645374 }, "gradients": { "norms": { "prediction": 0.0018048041965812445, "weighted_latent": 0.0014241763856261969, "total": 0.0014241763856261969 }, "prediction_weighted_latent_dot": 1.5527552932326216e-06, "prediction_weighted_latent_cosine": 0.6041004313564027, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0013200333341956139, "latent_mse": 0.01672578603029251, "scaled_prediction": 0.00019117724150419235, "scaled_weighted_latent": 0.0002422355319140479, "total": 0.0002422355319140479 }, "gradients": { "norms": { "prediction": 0.0024946476332843304, "weighted_latent": 0.0013707657344639301, "total": 0.0013707657344639301 }, "prediction_weighted_latent_dot": 1.628814516152488e-06, "prediction_weighted_latent_cosine": 0.47632039876463655, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.03142811357975006 }, { "step": 100, "losses": { "prediction_mse": 5.2756164222955704e-05, "latent_mse": 5.09874735143967e-05, "scaled_prediction": 7.640547664777841e-06, "scaled_weighted_latent": 7.384392688436492e-07, "total": 7.384392688436492e-07 }, "gradients": { "norms": { "prediction": 0.0006072220276109874, "weighted_latent": 8.430238085566089e-05, "total": 8.430238085566089e-05 }, "prediction_weighted_latent_dot": 2.7542473191033423e-08, "prediction_weighted_latent_cosine": 0.5380412557577855, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0007330543594434857 }, { "step": 200, "losses": { "prediction_mse": 4.1588969907024875e-05, "latent_mse": 7.497436005365898e-10, "scaled_prediction": 6.023230071150465e-06, "scaled_weighted_latent": 1.0858355881104842e-11, "total": 1.0858355881104842e-11 }, "gradients": { "norms": { "prediction": 0.00019014689314644784, "weighted_latent": 3.234289351894404e-07, "total": 3.234289351894404e-07 }, "prediction_weighted_latent_dot": 4.905533965149367e-11, "prediction_weighted_latent_cosine": 0.7976606761791432, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.771597794686386e-06 }, { "step": 300, "losses": { "prediction_mse": 4.151830944465473e-05, "latent_mse": 2.4699960162583544e-14, "scaled_prediction": 6.012996891513467e-06, "scaled_weighted_latent": 3.577235600260868e-16, "total": 3.577235600260868e-16 }, "gradients": { "norms": { "prediction": 0.00018826796440407634, "weighted_latent": 1.5522670926415572e-09, "total": 1.5522670926415572e-09 }, "prediction_weighted_latent_dot": -2.2286283875207896e-13, "prediction_weighted_latent_cosine": -0.7625964521082172, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.203239037295134e-07 }, { "step": 400, "losses": { "prediction_mse": 4.151829853071831e-05, "latent_mse": 2.7135230707916493e-14, "scaled_prediction": 6.012995072524063e-06, "scaled_weighted_latent": 3.92993006098131e-16, "total": 3.92993006098131e-16 }, "gradients": { "norms": { "prediction": 0.00018826920131687075, "weighted_latent": 1.6459560381321126e-09, "total": 1.6459560381321126e-09 }, "prediction_weighted_latent_dot": -2.2288270675688976e-13, "prediction_weighted_latent_cosine": -0.7192483290881527, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.580078929919182e-07 }, { "step": 500, "losses": { "prediction_mse": 4.151827670284547e-05, "latent_mse": 2.31092221232446e-14, "scaled_prediction": 6.012991889292607e-06, "scaled_weighted_latent": 3.346852961821743e-16, "total": 3.346852961821743e-16 }, "gradients": { "norms": { "prediction": 0.0001882680953713134, "weighted_latent": 1.5939622954874721e-09, "total": 1.5939622954874721e-09 }, "prediction_weighted_latent_dot": -2.3982706554188937e-13, "prediction_weighted_latent_cosine": -0.7991778167082637, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.1843847147474662e-07 }, { "step": 600, "losses": { "prediction_mse": 4.151826578890905e-05, "latent_mse": 2.3013358322406148e-14, "scaled_prediction": 6.0129905250505544e-06, "scaled_weighted_latent": 3.332969297722857e-16, "total": 3.332969297722857e-16 }, "gradients": { "norms": { "prediction": 0.0001882680953713134, "weighted_latent": 1.5846957079901358e-09, "total": 1.5846957079901358e-09 }, "prediction_weighted_latent_dot": -2.1274693432719177e-13, "prediction_weighted_latent_cosine": -0.7130840130378252, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.8792765388297994e-07 }, { "step": 700, "losses": { "prediction_mse": 4.151829853071831e-05, "latent_mse": 1.8244039363003034e-14, "scaled_prediction": 6.012995072524063e-06, "scaled_weighted_latent": 2.642240134318781e-16, "total": 2.642240134318781e-16 }, "gradients": { "norms": { "prediction": 0.00018826978339347988, "weighted_latent": 1.3986357627615575e-09, "total": 1.3986357627615575e-09 }, "prediction_weighted_latent_dot": -1.9212446032961655e-13, "prediction_weighted_latent_cosine": -0.7296211401274257, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.2540962163475342e-07 }, { "step": 800, "losses": { "prediction_mse": 4.151830944465473e-05, "latent_mse": 1.944334314862503e-14, "scaled_prediction": 6.012996891513467e-06, "scaled_weighted_latent": 2.8159324457849516e-16, "total": 2.8159324457849516e-16 }, "gradients": { "norms": { "prediction": 0.00018826867744792253, "weighted_latent": 1.471699984101349e-09, "total": 1.471699984101349e-09 }, "prediction_weighted_latent_dot": -2.0089251171872408e-13, "prediction_weighted_latent_cosine": -0.7250473870008983, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.597900845557888e-07 }, { "step": 900, "losses": { "prediction_mse": 4.1518345824442804e-05, "latent_mse": 1.809421109309501e-14, "scaled_prediction": 6.013001893734327e-06, "scaled_weighted_latent": 2.620541003094698e-16, "total": 2.620541003094698e-16 }, "gradients": { "norms": { "prediction": 0.00018827187886927277, "weighted_latent": 1.4837001627299173e-09, "total": 1.4837001627299173e-09 }, "prediction_weighted_latent_dot": -1.7204659463580796e-13, "prediction_weighted_latent_cosine": -0.6159060638539454, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.0148782431970176e-07 }, { "step": 1000, "losses": { "prediction_mse": 4.151833854848519e-05, "latent_mse": 2.2932690292641437e-14, "scaled_prediction": 6.013000984239625e-06, "scaled_weighted_latent": 3.321286331100056e-16, "total": 3.321286331100056e-16 }, "gradients": { "norms": { "prediction": 0.0001882705109892413, "weighted_latent": 1.6801082747264218e-09, "total": 1.6801082747264218e-09 }, "prediction_weighted_latent_dot": -1.919873494123786e-13, "prediction_weighted_latent_cosine": -0.6069501745436617, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.913382554150303e-07 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "d258f5bc6a790ad9fb7ae1657ed73565f958507c7180d00b83394cb390584ef2" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "fe1854be2dc714421e268893a4a5b66e5f1cdb3ca460feecff3d9c7b0247704a" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "e507f81cee389e9a0b0f12b9f39a92f8e280c0430f9b4f090b0a17f9cd40fed1" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "4b1f5ce0dbff7d1c13ff8f6ad98de19cf7724c14e93c60ff29c4b95c610bb8c8" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "55e0f47977e948cd79a17c6f943e37beaf5ba0919f38111cb4390847eae22c7f" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "b641c160f9124d96c7ab50831dc5f2d0b968c86c0db30c3c314183d3243ffa5b" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "77253c4219569e9dc5916087a4e12e5b8cf9b23b01dead990b0e7352c3d5f7c5" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "d6bef779f38b6a94fc6c878c359f57dfc74089c02e7aea9e6f9c8f5cc69a1469" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "4ea58a82a9c823fad458cdb3c8a1c76714a101dda72e02ee13960eab41b0d6c0" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "7550980e321a2bd7453351e6a8c216e9cd64bc018cee845fd1b672231788be1f" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "11aa8f9755fc7aec1e25e3e7a68f1427745d2c2ddd542cf7a418238615450c19" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "d72fb76a891400e98d3b3de0e8d37bce8f8b9352e6c9f04e76f7a9779a002ce8" } ], "final_losses": { "prediction_mse": 4.151833854848519e-05, "latent_mse": 2.2932690292641437e-14, "scaled_prediction": 6.013000984239625e-06, "scaled_weighted_latent": 3.321286331100056e-16, "total": 3.321286331100056e-16 }, "training_seconds": 2.756971299997531, "evaluation_seconds": 0.5017042999970727 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 0, "arm": "refit", "status": "passed", "files": { "seed-0-refit/result.json": "6f806bfaa84452d1cb3d7fe67b2e6286794bb6f36be206ef15928e348d8f537a", "seed-0-refit/checkpoint.json": "8adc1ee89716b4e854dfa3bbec62ab6fc88748dff57801c330b170279a9afd29", "inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "seed-0-refit/diagnostics-arrays.npz": "115f24fd9bb08c96ca7ba5897a19c1dc1d81acc5671784d80d7809436b0bc325" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642", "parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182", "parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 3.8743019104003906e-07, "max_tolerance_ratio": 0.35905024250805767, "worst_tolerance_index": [ 64, 7 ], "actual_at_worst": 0.0007908046245574951, "saved_at_worst": 0.0007904171943664551, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 4.15187023463659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 4.151866596657783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6229368565750986e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444363291142508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 7.444350922014564e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.0001227772822666739 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926477193832397, "saved_at_worst": 1.2926477193832397, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4581003251396934, "worst_tolerance_index": [ 62, 8 ], "actual_at_worst": -0.0012227296829223633, "saved_at_worst": -0.0012222155928611755, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 3.652342638815753e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014773458242416 }, "reported": { "count": 784, "observable_mse": 3.652340092230588e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014773458242416 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5373180166452665e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 3.181101055815816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004940252751111984 }, "reported": { "count": 580, "observable_mse": 3.181101055815816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004940252751111984 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 4.129161607124843e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014772526919842 }, "reported": { "count": 112, "observable_mse": 4.129156150156632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005014772526919842 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.4345281951175e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 6.04273664066568e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "reported": { "count": 92, "observable_mse": 6.0427275457186624e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005461216438561678 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-11, "numeric_tolerance_ratio": 9.040318834088458e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.778126847464591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 98, "observable_mse": 3.7781326682306826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987823776900768 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003417623690158127 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.11751791741699e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0049878195859491825 }, "reported": { "count": 70, "observable_mse": 3.1175259209703654e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987821914255619 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.001553465506768622 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.356186400400475e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987818654626608 }, "reported": { "count": 14, "observable_mse": 4.356192584964447e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498782517388463 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004349702472956356 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004987813998013735 }, "reported": { "count": 14, "observable_mse": 6.503110489575192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00498783215880394 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.816079020500183e-08, "numeric_tolerance_ratio": 0.012117022670509476 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7222198443487287e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297571886330843 }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529756722971797 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0030440218389961885 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.335614383104257e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00529757235199213 }, "reported": { "count": 70, "observable_mse": 3.3356191124767065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003652826317988257 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0065819373703562e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297572817653418 }, "reported": { "count": 14, "observable_mse": 3.0065733881201595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297566764056683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003957228511153945 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.370883056661114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052975681610405445 }, "reported": { "count": 14, "observable_mse": 6.370881601469591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005297568626701832 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003044021561014145 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7371144571807235e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311058834195137 }, "reported": { "count": 98, "observable_mse": 3.737111546797678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311060231178999 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0009124017806934365 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.151369674014859e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311061628162861 }, "reported": { "count": 70, "observable_mse": 3.151365672238171e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311059299856424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0015206697269865119 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.960085814469494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311049986630678 }, "reported": { "count": 14, "observable_mse": 3.960098547395319e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311064422130585 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4435499906539917e-08, "numeric_tolerance_ratio": 0.009428149153154155 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.442864105338231e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311055574566126 }, "reported": { "count": 14, "observable_mse": 6.442856829380617e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005311062093824148 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004257874458583739 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.720241511473432e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "reported": { "count": 98, "observable_mse": 3.720237145898864e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681416642266 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.12783413392026e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518278449774 }, "reported": { "count": 72, "observable_mse": 3.127835225313902e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005473518744111061 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00030094078470998165 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.02617733925581e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254744619131 }, "reported": { "count": 14, "observable_mse": 4.0261500544147566e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005477254278957844 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003008681507163908 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.917762220837176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499614402651787 }, "reported": { "count": 12, "observable_mse": 6.917752762092277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005499619524925947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0033047741282601615 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.71916830772534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498461425304413 }, "reported": { "count": 98, "observable_mse": 3.719156302395277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498470738530159 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.006009125611987981 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 3.126173032796942e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0054947249591350555 }, "reported": { "count": 72, "observable_mse": 3.126166484435089e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00549473287537694 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.916241884231567e-09, "numeric_tolerance_ratio": 0.005108988937015792 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.025795715278946e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498459096997976 }, "reported": { "count": 14, "observable_mse": 4.0257600630866364e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005498471204191446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.210719347000122e-08, "numeric_tolerance_ratio": 0.007811863060872042 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 6.919406587257981e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055208331905305386 }, "reported": { "count": 12, "observable_mse": 6.919389124959707e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005520846229046583 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.30385160446167e-08, "numeric_tolerance_ratio": 0.008400647652971193 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 3.75562631234061e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 98, "observable_mse": 3.755628131330013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 3.164217559969984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217758007347584 }, "reported": { "count": 70, "observable_mse": 3.164222289342433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002141989845922839 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0052177575416862965 }, "reported": { "count": 14, "observable_mse": 3.9748207200318575e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0024479883953403875 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 6.49347566650249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217755679041147 }, "reported": { "count": 14, "observable_mse": 6.493463297374547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005217761266976595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0036719825930105815 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622227424988523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.184563972055912e-11, "numeric_tolerance_ratio": 6.156109073935996e-05 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 4.151871689828113e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "reported": { "count": 84, "observable_mse": 4.151866596657783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.072111599205138e-05 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.444363291142508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360026644082518e-16 }, "reported": { "count": 14, "observable_mse": 7.444350922014564e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360023997104558e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.0001227772822666739 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640167688019574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707476753741503 }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005929166943850281 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 2.1136473151273094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005575723946094513 }, "reported": { "count": 72, "observable_mse": 2.1136505893082358e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055757262744009495 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0014948300936472665 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.2392847313312814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707480479031801 }, "reported": { "count": 14, "observable_mse": 2.239291643491015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005707477685064077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0017787500831550841 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.3784225049894303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441648583859205 }, "reported": { "count": 12, "observable_mse": 2.37841213674983e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006441639736294746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.005381193482372771 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2945678234100342, "saved_at_worst": 1.2945678234100342, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5731212112389529, "worst_tolerance_index": [ 1729, 8 ], "actual_at_worst": -0.0003994107246398926, "saved_at_worst": -0.0004000067710876465, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 3.751116673811339e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009355312213301659 }, "reported": { "count": 6272, "observable_mse": 3.751118129002862e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009355313144624233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00048117153551408443 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 3.3697288017719984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009192988276481628 }, "reported": { "count": 4092, "observable_mse": 3.369730256963521e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009192988276481628 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.450304388257766e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 4.0343762520933524e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009355311281979084 }, "reported": { "count": 896, "observable_mse": 4.034377707284875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009355312213301659 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00048117155866669024 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 4.7689049097243696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.009854805655777454 }, "reported": { "count": 1284, "observable_mse": 4.768906001118012e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.009854805655777454 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0862135916119795e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2790601253509521, "saved_at_worst": 1.2790601253509521, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.003545761108398e-07, "max_tolerance_ratio": 0.5058330479705814, "worst_tolerance_index": [ 1528, 4 ], "actual_at_worst": -0.0008991509675979614, "saved_at_worst": -0.0008997023105621338, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 8.092550706351176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018027817830443382 }, "reported": { "count": 3136, "observable_mse": 8.092560165096074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018027817830443382 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.382813912334983e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 7.759645086480305e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017672287300229073 }, "reported": { "count": 1582, "observable_mse": 7.759651634842157e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017672287300229073 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.548361852765083e-11, "numeric_tolerance_ratio": 6.497940101235427e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 8.137684926623479e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018027817830443382 }, "reported": { "count": 448, "observable_mse": 8.137695112964138e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018027817830443382 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.00010104116440875068 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 8.550452184863389e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01852450706064701 }, "reported": { "count": 1106, "observable_mse": 8.55046309879981e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01852450706064701 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-10, "numeric_tolerance_ratio": 0.00010821408368344566 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2832953929901123, "saved_at_worst": 1.2832953929901123, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.491185890233412, "worst_tolerance_index": [ 14, 4 ], "actual_at_worst": 0.002135440707206726, "saved_at_worst": 0.002134844660758972, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00030053246882744133, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03361130878329277 }, "reported": { "count": 3136, "observable_mse": 0.0003005327598657459, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03361130878329277 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002825468462187876 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00030422108829952776, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03238679841160774 }, "reported": { "count": 1001, "observable_mse": 0.0003042213211301714, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.032386794686317444 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0008788799261729612 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0002996882249135524, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03361131250858307 }, "reported": { "count": 448, "observable_mse": 0.0002996884868480265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03361131250858307 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.6193447411060333e-10, "numeric_tolerance_ratio": 0.00025431300611185976 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.0002985680184792727, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03431724011898041 }, "reported": { "count": 1687, "observable_mse": 0.00029856833862140775, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.03431724011898041 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2014213502407074e-10, "numeric_tolerance_ratio": 0.00031086081530719425 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2083498239517212, "saved_at_worst": 1.2083498239517212, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.364418029785156e-07, "max_tolerance_ratio": 0.40516607583806, "worst_tolerance_index": [ 2102, 7 ], "actual_at_worst": -0.0002973973751068115, "saved_at_worst": -0.00029781460762023926, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0013533851597458124, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06821805983781815 }, "reported": { "count": 3136, "observable_mse": 0.0013533857418224216, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06821805983781815 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.00051268989037383 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0013032231945544481, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06432659178972244 }, "reported": { "count": 407, "observable_mse": 0.001303224009461701, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06432659178972244 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.149072527885437e-10, "numeric_tolerance_ratio": 0.0007209511658854158 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0013566911220550537, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06821805238723755 }, "reported": { "count": 448, "observable_mse": 0.0013566919369623065, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06821804493665695 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0009525398650602308 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0013616863870993257, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0688893124461174 }, "reported": { "count": 2281, "observable_mse": 0.0013616869691759348, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0688893124461174 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005123153020443514 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128660127520561 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.758500679396093e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7585035897791386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.899485319690801e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010128450579941273 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7913549022050574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.7913512642262504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6242380475881334e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010601093992590904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.960090907639824e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.960088361054659e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.536540241476581e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010598846711218357 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.98074698750861e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.980756446253508e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.42124123167376e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03354682773351669 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00016841452452354133, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001684145099716261, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4310898925389558e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.034877024590969086 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002028434246312827, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00020284306083340198, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-10, "numeric_tolerance_ratio": 0.00035656520299299306 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.033779799938201904 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018723947869148105, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018723943503573537, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.285336176055767e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020691297948360443 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.837881275918335e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.837849989300594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.00031043304973434846 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 6.998432908560917e-16 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.009684504009783268 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.35564543295186e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.35564543295186e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.622226697392762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.622221240424551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.431860947590584e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0048739295452833176 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.30572659347672e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.3057307771523483e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.1740513894015114e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.1640156774083152e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.1640189515892416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.2671108366151636e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010431777685880661 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.1628161600092426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.162837260286324e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1100277081131935e-10, "numeric_tolerance_ratio": 0.0002099189932115251 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.72222057194449e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.722221663338132e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0873462981808747e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.857536648865789e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.8575337384827435e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8991992866106846e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7742513086413965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.774241849896498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.423179539860027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.7556277675321326e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.755628131330013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6243670323069163e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.1641156713012606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.1641131247160956e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5360248705162758e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 4.222623465466313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 4.222628194838762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.709486040680649e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 3.188768096151762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 3.1887459044810385e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.219167072325945e-10, "numeric_tolerance_ratio": 0.0002212113205401971 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1726252927910537e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.172617289237678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.978241468780203e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 5.0208916945848614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.020887329010293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.343765013792109e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 4.151866596657783e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 6.013048277964117e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 6.013048277964117e-06 }, "gradients": { "norms": { "prediction": 0.00018827685562428087, "weighted_latent": 2.6831793128829187e-18, "total": 0.00018827685562428087 }, "prediction_weighted_latent_dot": 8.642760077601405e-27, "prediction_weighted_latent_cosine": 1.710825928725321e-05, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "8adc1ee89716b4e854dfa3bbec62ab6fc88748dff57801c330b170279a9afd29" } ], "final_losses": { "prediction_mse": 4.151866596657783e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 6.013048277964117e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 6.013048277964117e-06 }, "training_seconds": 4.309997893869877e-05, "evaluation_seconds": 0.6919727000058629 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 1, "arm": "prediction_latent", "status": "passed", "files": { "seed-1-prediction_latent/result.json": "b9acf2d8ca134acac2d22d8ea5c94f40b34f9c9f8f377fdf91c019b6d88fc5dc", "seed-1-prediction_latent/checkpoint.json": "8839c5da214cb000de1e7257c4b9ecc65d0f72b253308ad9faea8bf91a8fdc6a", "inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "seed-1-prediction_latent/diagnostics-arrays.npz": "77a8b37898db44ded5f133b42e8963446d39beabecdba15b4818753f4c494cef" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.6193599700927734e-07, "max_tolerance_ratio": 0.45109829629777515, "worst_tolerance_index": [ 1, 6 ], "actual_at_worst": -0.00023978948593139648, "saved_at_worst": -0.00024025142192840576, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676346371648833e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 1.8676328181754798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.815598532142596e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.369931800989434e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 3.369917976669967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.00013777889110753617 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.291385531425476, "saved_at_worst": 1.291385531425476, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.5049480542195207, "worst_tolerance_index": [ 616, 3 ], "actual_at_worst": 0.0009182542562484741, "saved_at_worst": 0.0009188055992126465, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 1.4954192010918632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004067175090312958 }, "reported": { "count": 784, "observable_mse": 1.4954184734961018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004067175090312958 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.265093259511481e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.2249108294781763e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004047949332743883 }, "reported": { "count": 580, "observable_mse": 1.2249111023265868e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004047949332743883 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-12, "numeric_tolerance_ratio": 2.7251460436744265e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 1.770996823324822e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00406717648729682 }, "reported": { "count": 112, "observable_mse": 1.770994822436478e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00406717648729682 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9973510455402024e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 2.8653117624344304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "reported": { "count": 92, "observable_mse": 2.8653077606577426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.001776687800884e-11, "numeric_tolerance_ratio": 3.990343126672343e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.591147156432271e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205137968063354 }, "reported": { "count": 98, "observable_mse": 1.591144609847106e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120511934161186 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0013191059629536056 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2873621926701162e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120512865483761 }, "reported": { "count": 70, "observable_mse": 1.28735900943866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205123998224735 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00032977647986314834 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.774524753272999e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120515659451485 }, "reported": { "count": 14, "observable_mse": 1.7745223885867745e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120510071516037 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003957318410873048 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9266941055539064e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00412051472812891 }, "reported": { "count": 14, "observable_mse": 2.9266951969475485e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120514262467623 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003297764363621434 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6111895092763007e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151817861944437 }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151880726218224 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004550374085107517 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.384819915983826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815179457888007 }, "reported": { "count": 70, "observable_mse": 1.3848208254785277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.0065727623458328764 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4024532902112696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "reported": { "count": 14, "observable_mse": 1.402454017807031e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815184347331524 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0030335835414162663 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9517736038542353e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151845801621675 }, "reported": { "count": 14, "observable_mse": 2.9517867005779408e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151901680976152 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004044776351031832 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6099873391794972e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 98, "observable_mse": 1.609983155503869e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410440331324935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0029713781524049092 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2653170415433124e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 70, "observable_mse": 1.2653120393224526e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104403778910637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0033015311714488147 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.9126227925880812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "reported": { "count": 14, "observable_mse": 1.9126269762637094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104407504200935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.007923672718656958 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0307024644571356e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041044000536203384 }, "reported": { "count": 14, "observable_mse": 3.0306960979942232e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0026412263323736225 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384033847600222 }, "reported": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384037107229233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022661433552030583 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2924613656650763e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374961368739605 }, "reported": { "count": 72, "observable_mse": 1.2924631846544798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374966491013765 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003563329461384987 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7241960449609905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043840352445840836 }, "reported": { "count": 14, "observable_mse": 1.724195317365229e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384040366858244 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.00356108160832695 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.179249688400887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0044380780309438705 }, "reported": { "count": 12, "observable_mse": 3.179239502060227e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004438071511685848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004515324651932781 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5865916793700308e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355667158961296 }, "reported": { "count": 98, "observable_mse": 1.5865951354498975e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355666693300009 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003243745464814044 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2939838597958442e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043465327471494675 }, "reported": { "count": 72, "observable_mse": 1.2939861335325986e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004346531815826893 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006491621714371808 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7255477359867655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355668555945158 }, "reported": { "count": 14, "observable_mse": 1.725551010167692e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355671815574169 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022706210151850026 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.180122803314589e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004410072695463896 }, "reported": { "count": 12, "observable_mse": 3.1801344448467717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00441007548943162 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001938898741991697 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6066007447079755e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 98, "observable_mse": 1.6065991076175123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838496565819 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0036308001127401722 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2739343219436705e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843153178692 }, "reported": { "count": 70, "observable_mse": 1.2739328667521477e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838962227106 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002970654539643297 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.8545022612670437e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107845015823841 }, "reported": { "count": 14, "observable_mse": 1.854493712016847e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107835702598095 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00660145605781308 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.022031523869373e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 14, "observable_mse": 3.0220360713428818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107840359210968 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0023105088575984434 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6136897102114744e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003715944942086935 }, "reported": { "count": 98, "observable_mse": 1.6136866179294884e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037159454077482224 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003395036021685274 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 1.3835786376148462e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037159451749175787 }, "reported": { "count": 84, "observable_mse": 1.3835753634339198e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003715945640578866 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003395035964053927 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.994356327690184e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003715943545103073 }, "reported": { "count": 14, "observable_mse": 2.9943543268018402e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037159447092562914 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0008487590486448316 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897396527579986e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00388339813798666 }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834072183817625 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.006540465866677444 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 6.000454959576018e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964682560414076 }, "reported": { "count": 72, "observable_mse": 6.000467237754492e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964771036058664 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006412915697540378 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.797699254297186e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038833962753415108 }, "reported": { "count": 14, "observable_mse": 7.79771198722301e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834111765027046 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.010733069131502397 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 1.6882830095710233e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043688067235052586 }, "reported": { "count": 12, "observable_mse": 1.6882742784218863e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004368808586150408 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.001296311477784105 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2916944026947021, "saved_at_worst": 1.2916944026947021, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.487018426439568, "worst_tolerance_index": [ 3093, 3 ], "actual_at_worst": 0.0007083266973495483, "saved_at_worst": 0.000708848237991333, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 1.5423491277033463e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007828881964087486 }, "reported": { "count": 6272, "observable_mse": 1.5423493096022867e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007828881964087486 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-12, "numeric_tolerance_ratio": 1.8161882069188074e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.2795794646081049e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0077840243466198444 }, "reported": { "count": 4092, "observable_mse": 1.2795802831533365e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0077840243466198444 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.185452315956354e-12, "numeric_tolerance_ratio": 8.174991757688275e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 1.6711635907995515e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007828882895410061 }, "reported": { "count": 896, "observable_mse": 1.67116286320379e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007828882895410061 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.263818590310251e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 2.289883923367597e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007970155216753483 }, "reported": { "count": 1284, "observable_mse": 2.289882831973955e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.007970155216753483 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.088900188280637e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2809293270111084, "saved_at_worst": 1.2809293270111084, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5258328609838328, "worst_tolerance_index": [ 1761, 6 ], "actual_at_worst": -0.0004856884479522705, "saved_at_worst": -0.00048513710498809814, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.089183519477956e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01538029033690691 }, "reported": { "count": 3136, "observable_mse": 3.089184610871598e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01538029033690691 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0880325088450617e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 2.921657869592309e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.015239685773849487 }, "reported": { "count": 1582, "observable_mse": 2.9216605980764143e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.015239685773849487 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-11, "numeric_tolerance_ratio": 2.720535623581703e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.089471283601597e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.015380291268229485 }, "reported": { "count": 448, "observable_mse": 3.089472374995239e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01538029033690691 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00036694717131001057 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 3.328692037030123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.015579203143715858 }, "reported": { "count": 1106, "observable_mse": 3.3286916732322425e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.015579202212393284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0003640936753548345 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2789326906204224, "saved_at_worst": 1.2789326906204224, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.44440771948250757, "worst_tolerance_index": [ 3009, 12 ], "actual_at_worst": 0.0003948807716369629, "saved_at_worst": 0.0003944188356399536, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0001303740864386782, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030089888721704483 }, "reported": { "count": 3136, "observable_mse": 0.00013037415919825435, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030089886859059334 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.00046461721275974235 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00013071941793896258, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0300093162804842 }, "reported": { "count": 1001, "observable_mse": 0.0001307194761466235, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0300093162804842 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.745659136107834e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0001292203669436276, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030089884996414185 }, "reported": { "count": 448, "observable_mse": 0.0001292204688070342, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030089884996414185 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.00010056391497475709 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00013047554239165038, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030137594789266586 }, "reported": { "count": 1687, "observable_mse": 0.00013047562970314175, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.030137594789266586 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.618696156200086e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.254866123199463, "saved_at_worst": 1.254866123199463, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.748603820800781e-07, "max_tolerance_ratio": 0.41287302871708076, "worst_tolerance_index": [ 2452, 3 ], "actual_at_worst": -0.0008269697427749634, "saved_at_worst": -0.0008274167776107788, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0006314623169600964, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05984501913189888 }, "reported": { "count": 3136, "observable_mse": 0.00063146254979074, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05984501540660858 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.000533365233979093 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0006536822183988988, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05909571051597595 }, "reported": { "count": 407, "observable_mse": 0.0006536825094372034, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05909571051597595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00027318094406279765 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0006347591406665742, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.059845007956027985 }, "reported": { "count": 448, "observable_mse": 0.0006347593734972179, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.059845007956027985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.000218933626494742 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0006268501165322959, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.059977732598781586 }, "reported": { "count": 2281, "observable_mse": 0.0006268503493629396, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.059977736324071884 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0005323536447663624 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008089635521173477 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00808099564164877 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.650881858950015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.650880221859552e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6343922773059064e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008295925334095955 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6665429939166643e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6665442672092468e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2711741145488733e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008289745077490807 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5941945093800314e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5941997844493017e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.266673141096683e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.027659518644213676 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.701061935629696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.701080853119493e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8917489796876907e-10, "numeric_tolerance_ratio": 0.00018735732936814417 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025488151237368584 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.793230906827375e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.793223630869761e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.219692932613361e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025727983564138412 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.805288234725595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.805289689917117e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4424899806819776e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01637626811861992 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.188980554114096e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.188985647284426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0769799138515745e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007163230329751968 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.8452581571182236e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.845252154453192e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.002665031701326e-11, "numeric_tolerance_ratio": 5.991609002281226e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01430405955761671 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.9898714021546766e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.989876131527126e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.7105778271808575e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0038363952189683914 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.709371286386158e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.709345820534509e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5443691858507307e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897401075053494e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-13, "numeric_tolerance_ratio": 4.5440247119973885e-07 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008749740198254585 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0526686057564802e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0526651496766135e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.4490002241757354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.611191146366764e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.080316883302202e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5991054169717245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5991059626685455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.448255872186213e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6771049558883533e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.677106411079876e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4527550979481874e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6873334971023723e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6873336790013127e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-12, "numeric_tolerance_ratio": 1.8159253315753377e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.9213950508856215e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.921396869875025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8155011054046775e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 1.764368607837241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.7643691535340622e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4473570618693215e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 7.5585703598335385e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 7.558583092759363e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2723308806138392e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.499810635636095e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.4998324635089375e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.181151457204371e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6655649233143777e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6655654690111987e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.447894385869645e-06 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0008061017724685371, "latent_mse": 0.01724092662334442, "scaled_prediction": 0.00011674577399389818, "scaled_weighted_latent": 0.000249696196988225, "total": 0.00036644196370616555 }, "gradients": { "norms": { "prediction": 0.0013187306467443705, "weighted_latent": 0.0013792344834655523, "total": 0.0023915665224194527 }, "prediction_weighted_latent_dot": 1.0391254363639746e-06, "prediction_weighted_latent_cosine": 0.571312557453286, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0008706919034011662, "latent_mse": 0.01624198444187641, "scaled_prediction": 0.0001261002034880221, "scaled_weighted_latent": 0.00023522874107584357, "total": 0.00036132894456386566 }, "gradients": { "norms": { "prediction": 0.0017319160979241133, "weighted_latent": 0.0013384808553382754, "total": 0.0026371318381279707 }, "prediction_weighted_latent_dot": 1.0817008160302066e-06, "prediction_weighted_latent_cosine": 0.46662515227534174, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.029491163790225983 }, { "step": 100, "losses": { "prediction_mse": 1.2324222552706487e-05, "latent_mse": 7.64099313528277e-05, "scaled_prediction": 1.7848874449555296e-06, "scaled_weighted_latent": 1.106626655200671e-06, "total": 2.8915142138430383e-06 }, "gradients": { "norms": { "prediction": 5.4994736274238676e-05, "weighted_latent": 9.689817670732737e-05, "total": 0.00012219359632581472 }, "prediction_weighted_latent_dot": 1.2588040609884388e-09, "prediction_weighted_latent_cosine": 0.2362225788337237, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0007766924099996686 }, { "step": 200, "losses": { "prediction_mse": 1.3779517757939175e-05, "latent_mse": 1.353805600956548e-05, "scaled_prediction": 1.9956544292654144e-06, "scaled_weighted_latent": 1.9606839885000227e-07, "total": 2.1917228423262713e-06 }, "gradients": { "norms": { "prediction": 3.913431282853708e-05, "weighted_latent": 3.868190469802357e-05, "total": 5.253093036117207e-07 }, "prediction_weighted_latent_dot": -1.5137540110288228e-09, "prediction_weighted_latent_cosine": -0.9999763849457488, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 6.9785883169970475e-06 }, { "step": 300, "losses": { "prediction_mse": 1.3765730727754999e-05, "latent_mse": 1.3676089110958856e-05, "scaled_prediction": 1.993657633647672e-06, "scaled_weighted_latent": 1.980674966262086e-07, "total": 2.1917251160630258e-06 }, "gradients": { "norms": { "prediction": 3.886024933308363e-05, "weighted_latent": 3.890350490109995e-05, "total": 1.0165747426071903e-06 }, "prediction_weighted_latent_dot": -1.5112843199105441e-09, "prediction_weighted_latent_cosine": -0.9996589624877547, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.979640991427004e-05 }, { "step": 400, "losses": { "prediction_mse": 1.3768789358437061e-05, "latent_mse": 1.3674572983291e-05, "scaled_prediction": 1.9941005575674353e-06, "scaled_weighted_latent": 1.980455408556736e-07, "total": 2.192145984736271e-06 }, "gradients": { "norms": { "prediction": 3.891400410793722e-05, "weighted_latent": 3.8907051930436864e-05, "total": 5.7175780057150405e-06 }, "prediction_weighted_latent_dot": -1.4976839768365835e-09, "prediction_weighted_latent_cosine": -0.9892041698781848, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0003822461876552552 }, { "step": 500, "losses": { "prediction_mse": 1.3779789696855005e-05, "latent_mse": 1.3661119737662375e-05, "scaled_prediction": 1.995693764911266e-06, "scaled_weighted_latent": 1.978507100375282e-07, "total": 2.1935445602139225e-06 }, "gradients": { "norms": { "prediction": 4.00488315790426e-05, "weighted_latent": 3.890451989718713e-05, "total": 1.3405261597654317e-05 }, "prediction_weighted_latent_dot": -1.4688850136224119e-09, "prediction_weighted_latent_cosine": -0.9427529272839232, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005967266042716801 }, { "step": 600, "losses": { "prediction_mse": 1.376701857225271e-05, "latent_mse": 1.3671409760718234e-05, "scaled_prediction": 1.9938440800615354e-06, "scaled_weighted_latent": 1.979997250600718e-07, "total": 2.191843805121607e-06 }, "gradients": { "norms": { "prediction": 3.883298268192448e-05, "weighted_latent": 3.890189327648841e-05, "total": 3.6848484796792036e-06 }, "prediction_weighted_latent_dot": -1.5038899014996332e-09, "prediction_weighted_latent_cosine": -0.99550754500707, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00015977371367625892 }, { "step": 700, "losses": { "prediction_mse": 1.3792460777040105e-05, "latent_mse": 1.370991321891779e-05, "scaled_prediction": 1.9975288978457684e-06, "scaled_weighted_latent": 1.9855737320995104e-07, "total": 2.1960863705317024e-06 }, "gradients": { "norms": { "prediction": 4.25968537456356e-05, "weighted_latent": 3.8942300307098776e-05, "total": 1.801659345801454e-05 }, "prediction_weighted_latent_dot": -1.5031985656221991e-09, "prediction_weighted_latent_cosine": -0.9061857496691742, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0010927342809736729 }, { "step": 800, "losses": { "prediction_mse": 1.3770555597147904e-05, "latent_mse": 1.3717112778977025e-05, "scaled_prediction": 1.994356352952309e-06, "scaled_weighted_latent": 1.9866163825099648e-07, "total": 2.193017962781596e-06 }, "gradients": { "norms": { "prediction": 4.083912426722236e-05, "weighted_latent": 3.893457324011251e-05, "total": 1.1470227946119849e-05 }, "prediction_weighted_latent_dot": -1.5260845920295196e-09, "prediction_weighted_latent_cosine": -0.9597691098227897, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00030544379842467606 }, { "step": 900, "losses": { "prediction_mse": 1.3786719136987813e-05, "latent_mse": 1.3740334907197393e-05, "scaled_prediction": 1.9966973923146725e-06, "scaled_weighted_latent": 1.9899795233868645e-07, "total": 2.19569528780994e-06 }, "gradients": { "norms": { "prediction": 4.388808520161547e-05, "weighted_latent": 3.902985918102786e-05, "total": 2.101091740769334e-05 }, "prediction_weighted_latent_dot": -1.5040174661251626e-09, "prediction_weighted_latent_cosine": -0.8780298122504341, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0008061519474722445 }, { "step": 1000, "losses": { "prediction_mse": 1.3835753634339198e-05, "latent_mse": 1.3808251424052287e-05, "scaled_prediction": 2.003798954319791e-06, "scaled_weighted_latent": 1.9998157085865387e-07, "total": 2.203780468335026e-06 }, "gradients": { "norms": { "prediction": 5.0987840950256214e-05, "weighted_latent": 3.904912227881141e-05, "total": 3.4724351280601695e-05 }, "prediction_weighted_latent_dot": -1.4594064845496746e-09, "prediction_weighted_latent_cosine": -0.732990545077887, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.002056598663330078 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "89d8bc5ffec17a479dca72c2881e928d8e51bc09264d06412d3f75d556c67e74" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "7f42839fc28475d9ffa07003e56fad1804e51cf3b4be6b6e04200f48c186456e" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "ea8c90d58857fa07b3884aacfe5d8ec9dddd3d6b71ed9c956285ef82f083164f" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "b0f9b3b30a10095087dddac9300d70691ce6aa8ffc51f561d2970b3fed180937" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "1bba1bfc2961bd7f9a18824f1962222da0ae93b371f14697851cdab7a9cc63a2" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "d1deba08d32aa677cc5bfd6ac9abb98ce005268fa5b44f7684a14a5573620f15" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "4f4cd928216f757574214682a35b84bbe4cd4905ba3d4684c7c5d8209a3bf4fa" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "b0f88e8adc771a2547231245718b6a87d5af804ed5c3063e4f244ed935960ea7" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "aacf0c12fcddd8301ba9ef60f73597804bb76fef9ac6da035265d79c289c1319" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "bde9444c65c61fd4f7ce4fb4724126b712c0b6c56aa754a7e9cdb1c944d890cc" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "7272aa472e41b04e8c81c1a1279276555ef4b7cf9d430f8dbddb0d620e453738" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "8839c5da214cb000de1e7257c4b9ecc65d0f72b253308ad9faea8bf91a8fdc6a" } ], "final_losses": { "prediction_mse": 1.3835753634339198e-05, "latent_mse": 1.3808251424052287e-05, "scaled_prediction": 2.003798954319791e-06, "scaled_weighted_latent": 1.9998157085865387e-07, "total": 2.203780468335026e-06 }, "training_seconds": 3.2491507999948226, "evaluation_seconds": 0.5362058999598958 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 1, "arm": "latent_only", "status": "passed", "files": { "seed-1-latent_only/result.json": "9c59b3e4a5eb6a983a8866e9fbb5b08bfb3fe78276741c06bff9b2aa83c9ecc6", "seed-1-latent_only/checkpoint.json": "b23dac4e55edfd0ed3c5276331b5fd666cf6cb758202e1d9e5efe7d2d1aea453", "inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "seed-1-latent_only/diagnostics-arrays.npz": "846787664b0b27fac1ba45d574fb7d6cb7e9292bce0062d62f660bd361f1039f" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.6193599700927734e-07, "max_tolerance_ratio": 0.45109829629777515, "worst_tolerance_index": [ 1, 6 ], "actual_at_worst": -0.00023978948593139648, "saved_at_worst": -0.00024025142192840576, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676346371648833e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 1.8676328181754798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.815598532142596e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.369931800989434e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 3.369917976669967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.00013777889110753617 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.291385531425476, "saved_at_worst": 1.291385531425476, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.42046233387194104, "worst_tolerance_index": [ 87, 6 ], "actual_at_worst": 0.0009859204292297363, "saved_at_worst": 0.0009863823652267456, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 1.553986658109352e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003849141066893935 }, "reported": { "count": 784, "observable_mse": 1.5539857486146502e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003849140601232648 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00033623839970712183 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.2950129530509003e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037928984966129065 }, "reported": { "count": 580, "observable_mse": 1.29501286210143e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00379289872944355 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001688047220682108 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 1.817941119952593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038491422310471535 }, "reported": { "count": 112, "observable_mse": 1.8179387552663684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038491422310471535 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3603951707509615e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 2.8653117624344304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "reported": { "count": 92, "observable_mse": 2.8653077606577426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.001776687800884e-11, "numeric_tolerance_ratio": 3.990343126672343e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.591147156432271e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205137968063354 }, "reported": { "count": 98, "observable_mse": 1.591144609847106e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120511934161186 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0013191059629536056 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2873621926701162e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120512865483761 }, "reported": { "count": 70, "observable_mse": 1.28735900943866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205123998224735 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00032977647986314834 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.774524753272999e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120515659451485 }, "reported": { "count": 14, "observable_mse": 1.7745223885867745e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120510071516037 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003957318410873048 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9266941055539064e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00412051472812891 }, "reported": { "count": 14, "observable_mse": 2.9266951969475485e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120514262467623 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003297764363621434 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6111895092763007e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151817861944437 }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151880726218224 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004550374085107517 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.384819915983826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815179457888007 }, "reported": { "count": 70, "observable_mse": 1.3848208254785277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.0065727623458328764 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4024532902112696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "reported": { "count": 14, "observable_mse": 1.402454017807031e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815184347331524 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0030335835414162663 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9517736038542353e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151845801621675 }, "reported": { "count": 14, "observable_mse": 2.9517867005779408e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151901680976152 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004044776351031832 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6099873391794972e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 98, "observable_mse": 1.609983155503869e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410440331324935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0029713781524049092 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2653170415433124e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 70, "observable_mse": 1.2653120393224526e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104403778910637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0033015311714488147 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.9126227925880812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "reported": { "count": 14, "observable_mse": 1.9126269762637094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104407504200935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.007923672718656958 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0307024644571356e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041044000536203384 }, "reported": { "count": 14, "observable_mse": 3.0306960979942232e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0026412263323736225 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384033847600222 }, "reported": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384037107229233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022661433552030583 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2924613656650763e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374961368739605 }, "reported": { "count": 72, "observable_mse": 1.2924631846544798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374966491013765 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003563329461384987 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7241960449609905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043840352445840836 }, "reported": { "count": 14, "observable_mse": 1.724195317365229e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384040366858244 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.00356108160832695 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.179249688400887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0044380780309438705 }, "reported": { "count": 12, "observable_mse": 3.179239502060227e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004438071511685848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004515324651932781 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5865916793700308e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355667158961296 }, "reported": { "count": 98, "observable_mse": 1.5865951354498975e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355666693300009 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003243745464814044 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2939838597958442e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043465327471494675 }, "reported": { "count": 72, "observable_mse": 1.2939861335325986e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004346531815826893 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006491621714371808 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7255477359867655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355668555945158 }, "reported": { "count": 14, "observable_mse": 1.725551010167692e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355671815574169 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022706210151850026 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.180122803314589e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004410072695463896 }, "reported": { "count": 12, "observable_mse": 3.1801344448467717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00441007548943162 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001938898741991697 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6066007447079755e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 98, "observable_mse": 1.6065991076175123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838496565819 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0036308001127401722 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2739343219436705e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843153178692 }, "reported": { "count": 70, "observable_mse": 1.2739328667521477e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838962227106 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002970654539643297 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.8545022612670437e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107845015823841 }, "reported": { "count": 14, "observable_mse": 1.854493712016847e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107835702598095 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00660145605781308 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.022031523869373e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 14, "observable_mse": 3.0220360713428818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107840359210968 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0023105088575984434 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.082230093947146e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.6341662956165237e-07 }, "reported": { "count": 98, "observable_mse": 2.0822268197662197e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.6449087070213864e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.074241140486265e-09, "numeric_tolerance_ratio": 0.0010742234704908661 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676168110687286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.630751853554102e-07 }, "reported": { "count": 84, "observable_mse": 1.867613900685683e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.6483070908179798e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7555237263877643e-09, "numeric_tolerance_ratio": 0.0017554947904426545 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.369911064510234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.6545044445592794e-07 }, "reported": { "count": 14, "observable_mse": 3.3699041523505e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.6243686218331277e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.0135822726151673e-09, "numeric_tolerance_ratio": 0.0030135333217254806 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897396527579986e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00388339813798666 }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834072183817625 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.006540465866677444 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 6.000454959576018e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964682560414076 }, "reported": { "count": 72, "observable_mse": 6.000467237754492e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964771036058664 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006412915697540378 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.797699254297186e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038833962753415108 }, "reported": { "count": 14, "observable_mse": 7.79771198722301e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834111765027046 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.010733069131502397 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 1.6882830095710233e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043688067235052586 }, "reported": { "count": 12, "observable_mse": 1.6882742784218863e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004368808586150408 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.001296311477784105 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2916944026947021, "saved_at_worst": 1.2916944026947021, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.487018426439568, "worst_tolerance_index": [ 3093, 3 ], "actual_at_worst": 0.0007083266973495483, "saved_at_worst": 0.000708848237991333, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 1.5348035958595574e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00719467643648386 }, "reported": { "count": 6272, "observable_mse": 1.534803413960617e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00719467643648386 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-12, "numeric_tolerance_ratio": 1.8162018906835934e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.2756069736497011e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0070650274865329266 }, "reported": { "count": 4092, "observable_mse": 1.275607428397052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0070650274865329266 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-12, "numeric_tolerance_ratio": 4.541680107981497e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 1.6645597497699782e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007194678299129009 }, "reported": { "count": 896, "observable_mse": 1.664558658376336e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007194678299129009 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0895799723505444e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 2.2702946807839908e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007593102287501097 }, "reported": { "count": 1284, "observable_mse": 2.2702939531882294e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.007593102287501097 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2594764686533295e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2837709188461304, "saved_at_worst": 1.2837709188461304, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5258328609838328, "worst_tolerance_index": [ 1761, 6 ], "actual_at_worst": -0.0004856884479522705, "saved_at_worst": -0.00048513710498809814, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 2.7006264645024203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013975739479064941 }, "reported": { "count": 3136, "observable_mse": 2.700626828300301e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013975738547742367 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00038844374815022117 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 2.4990224119392224e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013708249665796757 }, "reported": { "count": 1582, "observable_mse": 2.4990240490296856e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013708249665796757 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6330095330957698e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 2.7068224881077185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013975740410387516 }, "reported": { "count": 448, "observable_mse": 2.7068224881077185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013975740410387516 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 2.986486470035743e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.014349685981869698 }, "reported": { "count": 1106, "observable_mse": 2.9864857424399815e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.014349685050547123 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.000382478283674783 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2804754972457886, "saved_at_worst": 1.2804754972457886, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4426357756470472, "worst_tolerance_index": [ 1822, 8 ], "actual_at_worst": 9.983032941818237e-05, "saved_at_worst": 9.938329458236694e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 9.646726539358497e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026145141571760178 }, "reported": { "count": 3136, "observable_mse": 9.646730177337304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026145141571760178 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.603219520606705e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 9.084730118047446e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02528628334403038 }, "reported": { "count": 1001, "observable_mse": 9.084734483622015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02528628334403038 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.326271540262718e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 9.527995280222967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026145141571760178 }, "reported": { "count": 448, "observable_mse": 9.527997463010252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026145141571760178 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.162185981706769e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00010011722042690963, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026641670614480972 }, "reported": { "count": 1687, "observable_mse": 0.00010011727863457054, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026641670614480972 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.7630678246279204e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2577427625656128, "saved_at_worst": 1.2577427625656128, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.556510925292969e-07, "max_tolerance_ratio": 0.43638591490078654, "worst_tolerance_index": [ 2430, 6 ], "actual_at_worst": -0.0014396607875823975, "saved_at_worst": -0.0014391615986824036, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00048254439025186, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05308973416686058 }, "reported": { "count": 3136, "observable_mse": 0.0004825446230825037, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05308973416686058 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00022211271406484442 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0004555753548629582, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.050318289548158646 }, "reported": { "count": 407, "observable_mse": 0.0004555756750050932, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.050318289548158646 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2014213502407074e-10, "numeric_tolerance_ratio": 0.00030619273866421015 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00048519056872464716, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05308973789215088 }, "reported": { "count": 448, "observable_mse": 0.00048519091797061265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05308973789215088 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.0003330849845397 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.00048683679779060185, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05356917530298233 }, "reported": { "count": 2281, "observable_mse": 0.00048683705972507596, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05356917157769203 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0005860215267881844 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008089635521173477 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00808099564164877 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.650881858950015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.650880221859552e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6343922773059064e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008295925334095955 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6665429939166643e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6665442672092468e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2711741145488733e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008289745077490807 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5941945093800314e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5941997844493017e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.266673141096683e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.027659518644213676 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.701061935629696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.701080853119493e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8917489796876907e-10, "numeric_tolerance_ratio": 0.00018735732936814417 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025488151237368584 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.793230906827375e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.793223630869761e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.219692932613361e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025727983564138412 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.805288234725595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.805289689917117e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4424899806819776e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01637626811861992 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.188980554114096e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.188985647284426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0769799138515745e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 2.802309211347165e-07 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822144506382756e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.082210994558409e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.448898532294068e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007543846964836121 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.959134533535689e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.959132350748405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1785192766633875e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0038363952189683914 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.709371286386158e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.709345820534509e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5443691858507307e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897401075053494e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-13, "numeric_tolerance_ratio": 4.5440247119973885e-07 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008749740198254585 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0526686057564802e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0526651496766135e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.4490002241757354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.611191146366764e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.080316883302202e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5991054169717245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5991059626685455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.448255872186213e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6771049558883533e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.677106411079876e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4527550979481874e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.606598380021751e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6065961972344667e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1792860515718863e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.812363916542381e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.8123662812286057e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3604083001964056e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 1.764368607837241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.7643691535340622e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4473570618693215e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 7.5585703598335385e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 7.558583092759363e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2723308806138392e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.499810635636095e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.4998324635089375e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.181151457204371e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6655649233143777e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6655654690111987e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.447894385869645e-06 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0008061017724685371, "latent_mse": 0.01724092662334442, "scaled_prediction": 0.00011674577399389818, "scaled_weighted_latent": 0.000249696196988225, "total": 0.000249696196988225 }, "gradients": { "norms": { "prediction": 0.0013187306467443705, "weighted_latent": 0.0013792344834655523, "total": 0.0013792344834655523 }, "prediction_weighted_latent_dot": 1.0391254363639746e-06, "prediction_weighted_latent_cosine": 0.571312557453286, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0008845120901241899, "latent_mse": 0.016194390133023262, "scaled_prediction": 0.0001281017466681078, "scaled_weighted_latent": 0.0002345394459553063, "total": 0.0002345394459553063 }, "gradients": { "norms": { "prediction": 0.0017962348647415638, "weighted_latent": 0.0013357902644202113, "total": 0.0013357902644202113 }, "prediction_weighted_latent_dot": 1.1284744232398225e-06, "prediction_weighted_latent_cosine": 0.4703166184558535, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.03137466311454773 }, { "step": 100, "losses": { "prediction_mse": 2.6100775357917883e-05, "latent_mse": 4.739292853628285e-05, "scaled_prediction": 3.7801123653480317e-06, "scaled_weighted_latent": 6.863803605483554e-07, "total": 6.863803605483554e-07 }, "gradients": { "norms": { "prediction": 0.0003757460799533874, "weighted_latent": 8.124136365950108e-05, "total": 8.124136365950108e-05 }, "prediction_weighted_latent_dot": 1.79601880034852e-08, "prediction_weighted_latent_cosine": 0.5883546842546467, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0007346367347054183 }, { "step": 200, "losses": { "prediction_mse": 1.8727765564108267e-05, "latent_mse": 8.353190361631846e-10, "scaled_prediction": 2.7122971459903056e-06, "scaled_weighted_latent": 1.2097723864346044e-11, "total": 1.2097723864346044e-11 }, "gradients": { "norms": { "prediction": 0.0001373724517179653, "weighted_latent": 3.3851549119390256e-07, "total": 3.3851549119390256e-07 }, "prediction_weighted_latent_dot": 3.571216586339965e-11, "prediction_weighted_latent_cosine": 0.7679589267440738, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.8460998489899794e-06 }, { "step": 300, "losses": { "prediction_mse": 1.867616083472967e-05, "latent_mse": 1.5077970813635704e-14, "scaled_prediction": 2.7048233732784865e-06, "scaled_weighted_latent": 2.1837061275120558e-16, "total": 2.1837061275120558e-16 }, "gradients": { "norms": { "prediction": 0.00013575522461906075, "weighted_latent": 1.2317837860820191e-09, "total": 1.2317837860820191e-09 }, "prediction_weighted_latent_dot": -1.1001419773001148e-13, "prediction_weighted_latent_cosine": -0.6578966881980206, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 8.404920492921519e-08 }, { "step": 400, "losses": { "prediction_mse": 1.8676202671485953e-05, "latent_mse": 1.4480321306712014e-14, "scaled_prediction": 2.7048295123677235e-06, "scaled_weighted_latent": 2.097150080563405e-16, "total": 2.097150080563405e-16 }, "gradients": { "norms": { "prediction": 0.00013575679622590542, "weighted_latent": 1.1411173117537032e-09, "total": 1.1411173117537032e-09 }, "prediction_weighted_latent_dot": -7.508601487966893e-14, "prediction_weighted_latent_cosine": -0.4846934833921122, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.6925260126754438e-07 }, { "step": 500, "losses": { "prediction_mse": 1.867614992079325e-05, "latent_mse": 1.6872303386242585e-14, "scaled_prediction": 2.7048217816627584e-06, "scaled_weighted_latent": 2.443574909287389e-16, "total": 2.443574909287389e-16 }, "gradients": { "norms": { "prediction": 0.00013575528282672167, "weighted_latent": 1.2982581676368454e-09, "total": 1.2982581676368454e-09 }, "prediction_weighted_latent_dot": -1.0371863551393926e-13, "prediction_weighted_latent_cosine": -0.5884898711148955, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.282730946490119e-07 }, { "step": 600, "losses": { "prediction_mse": 1.8676179024623707e-05, "latent_mse": 1.7656993014513074e-14, "scaled_prediction": 2.7048258743889164e-06, "scaled_weighted_latent": 2.5572197316337225e-16, "total": 2.5572197316337225e-16 }, "gradients": { "norms": { "prediction": 0.00013575544289778918, "weighted_latent": 1.320952680572418e-09, "total": 1.320952680572418e-09 }, "prediction_weighted_latent_dot": -1.0743573457087929e-13, "prediction_weighted_latent_cosine": -0.5991067960757013, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.4546375243517105e-07 }, { "step": 700, "losses": { "prediction_mse": 1.8676191757549532e-05, "latent_mse": 1.64592867330296e-14, "scaled_prediction": 2.7048279207519954e-06, "scaled_weighted_latent": 2.3837587660412705e-16, "total": 2.3837587660412705e-16 }, "gradients": { "norms": { "prediction": 0.00013575569028034806, "weighted_latent": 1.238080749033088e-09, "total": 1.238080749033088e-09 }, "prediction_weighted_latent_dot": -8.547593785112467e-14, "prediction_weighted_latent_cosine": -0.508553750463921, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.2718136822040833e-07 }, { "step": 800, "losses": { "prediction_mse": 1.867614992079325e-05, "latent_mse": 1.865822830981502e-14, "scaled_prediction": 2.7048217816627584e-06, "scaled_weighted_latent": 2.7022262135499423e-16, "total": 2.7022262135499423e-16 }, "gradients": { "norms": { "prediction": 0.00013575420598499477, "weighted_latent": 1.422379436455401e-09, "total": 1.422379436455401e-09 }, "prediction_weighted_latent_dot": -1.1380973203786726e-13, "prediction_weighted_latent_cosine": -0.5894006926123972, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.1941859763028333e-07 }, { "step": 900, "losses": { "prediction_mse": 1.8676177205634303e-05, "latent_mse": 1.746145207050763e-14, "scaled_prediction": 2.704825647015241e-06, "scaled_weighted_latent": 2.528899979135663e-16, "total": 2.528899979135663e-16 }, "gradients": { "norms": { "prediction": 0.0001357544242637232, "weighted_latent": 1.3941990895105505e-09, "total": 1.3941990895105505e-09 }, "prediction_weighted_latent_dot": -1.1605610407156713e-13, "prediction_weighted_latent_cosine": -0.6131817216372834, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.4708339196877205e-07 }, { "step": 1000, "losses": { "prediction_mse": 1.867613900685683e-05, "latent_mse": 2.7169164541849394e-14, "scaled_prediction": 2.7048201900470303e-06, "scaled_weighted_latent": 3.934844704959957e-16, "total": 3.934844704959957e-16 }, "gradients": { "norms": { "prediction": 0.00013575352204497904, "weighted_latent": 1.9588655142399602e-09, "total": 1.9588655142399602e-09 }, "prediction_weighted_latent_dot": -1.1047034191276686e-13, "prediction_weighted_latent_cosine": -0.41542245860004423, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.4463152448770416e-07 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "0019be82f6c82ad2528fa20d684da6e7c3a34adf8757608d6bac0ed0628f84aa" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "cba955951f5dcde7b0f5c797baacd4d3fb058da239bf7ca514b6afa2c38e3abc" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "6657da4d89be5f5567c70b162592681239a2511b88a0fcc29da8dde9ad7e0132" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "6f03faf58ef0cf71999afed7d14cedffe5f59797c8dcd912b132c44c882191f9" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "f97a1925abbdaf55021492c08c5ef775fffa51dbff05d06b6e75ee134d0145a9" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "df0d72df6257828cc0819af9bacd2c9df95eaaa6a74baccdba5db2e53dc9b5a8" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "08780c1646a18a11d27d727fdbb2f939f9d192ce79c588f5489fe9af0181c688" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "9cbcc1a4b0d82c2f61d1830a66a11492fd695daa5af016f09de1fd7cae98be55" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "0c77d8e63f912c8fb6a623325cdc4fedb286245b917124d90252cee318b02d5e" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "550d7d9030a3f2b73c7b47825309a1ad1c78e3a75816c83d299127f4d7a2bb57" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "b2e4941568ee67bde30dfa33cba010547a395da32e84b7b8de2726ade7ce4a05" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "b23dac4e55edfd0ed3c5276331b5fd666cf6cb758202e1d9e5efe7d2d1aea453" } ], "final_losses": { "prediction_mse": 1.867613900685683e-05, "latent_mse": 2.7169164541849394e-14, "scaled_prediction": 2.7048201900470303e-06, "scaled_weighted_latent": 3.934844704959957e-16, "total": 3.934844704959957e-16 }, "training_seconds": 2.7909303000196815, "evaluation_seconds": 0.519522299990058 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 1, "arm": "refit", "status": "passed", "files": { "seed-1-refit/result.json": "29f717c7e993f537bed0c71ed1e26fb24ecdb13f87d0927b63e256d4ad9006e2", "seed-1-refit/checkpoint.json": "f27c7aa4ae44e1bd2421dfb90baa66d8366a995f651d65613378922892ca067a", "inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "seed-1-refit/diagnostics-arrays.npz": "27aa56e1165bd249fe85280a3393502cf51ea898344f423e11966d15c5a0481b" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.6193599700927734e-07, "max_tolerance_ratio": 0.45109829629777515, "worst_tolerance_index": [ 1, 6 ], "actual_at_worst": -0.00023978948593139648, "saved_at_worst": -0.00024025142192840576, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676346371648833e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 1.8676328181754798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.815598532142596e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.369931800989434e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 3.369917976669967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.00013777889110753617 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.291385531425476, "saved_at_worst": 1.291385531425476, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.45109829629777515, "worst_tolerance_index": [ 588, 6 ], "actual_at_worst": -0.00023978948593139648, "saved_at_worst": -0.00024025142192840576, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 1.5539890227955766e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003849141066893935 }, "reported": { "count": 784, "observable_mse": 1.5539879314019345e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003849140601232648 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00033623839970712183 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.2950155905855354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037928984966129065 }, "reported": { "count": 580, "observable_mse": 1.2950153177371249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00379289872944355 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001688047220682108 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 1.8179434846388176e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038491422310471535 }, "reported": { "count": 112, "observable_mse": 1.817940028558951e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038491422310471535 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.4498083220975305e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 2.8653117624344304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "reported": { "count": 92, "observable_mse": 2.8653077606577426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.001776687800884e-11, "numeric_tolerance_ratio": 3.990343126672343e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.591147156432271e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205137968063354 }, "reported": { "count": 98, "observable_mse": 1.591144609847106e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120511934161186 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0013191059629536056 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2873621926701162e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120512865483761 }, "reported": { "count": 70, "observable_mse": 1.28735900943866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205123998224735 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00032977647986314834 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.774524753272999e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120515659451485 }, "reported": { "count": 14, "observable_mse": 1.7745223885867745e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120510071516037 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003957318410873048 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9266941055539064e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00412051472812891 }, "reported": { "count": 14, "observable_mse": 2.9266951969475485e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120514262467623 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003297764363621434 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6111895092763007e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151817861944437 }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151880726218224 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004550374085107517 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.384819915983826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815179457888007 }, "reported": { "count": 70, "observable_mse": 1.3848208254785277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.0065727623458328764 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4024532902112696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "reported": { "count": 14, "observable_mse": 1.402454017807031e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815184347331524 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0030335835414162663 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9517736038542353e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151845801621675 }, "reported": { "count": 14, "observable_mse": 2.9517867005779408e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151901680976152 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004044776351031832 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6099873391794972e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 98, "observable_mse": 1.609983155503869e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410440331324935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0029713781524049092 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2653170415433124e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 70, "observable_mse": 1.2653120393224526e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104403778910637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0033015311714488147 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.9126227925880812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "reported": { "count": 14, "observable_mse": 1.9126269762637094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104407504200935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.007923672718656958 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0307024644571356e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041044000536203384 }, "reported": { "count": 14, "observable_mse": 3.0306960979942232e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0026412263323736225 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384033847600222 }, "reported": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384037107229233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022661433552030583 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2924613656650763e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374961368739605 }, "reported": { "count": 72, "observable_mse": 1.2924631846544798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374966491013765 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003563329461384987 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7241960449609905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043840352445840836 }, "reported": { "count": 14, "observable_mse": 1.724195317365229e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384040366858244 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.00356108160832695 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.179249688400887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0044380780309438705 }, "reported": { "count": 12, "observable_mse": 3.179239502060227e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004438071511685848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004515324651932781 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5865916793700308e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355667158961296 }, "reported": { "count": 98, "observable_mse": 1.5865951354498975e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355666693300009 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003243745464814044 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2939838597958442e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043465327471494675 }, "reported": { "count": 72, "observable_mse": 1.2939861335325986e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004346531815826893 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006491621714371808 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7255477359867655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355668555945158 }, "reported": { "count": 14, "observable_mse": 1.725551010167692e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355671815574169 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022706210151850026 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.180122803314589e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004410072695463896 }, "reported": { "count": 12, "observable_mse": 3.1801344448467717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00441007548943162 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001938898741991697 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6066007447079755e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 98, "observable_mse": 1.6065991076175123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838496565819 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0036308001127401722 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2739343219436705e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843153178692 }, "reported": { "count": 70, "observable_mse": 1.2739328667521477e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838962227106 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002970654539643297 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.8545022612670437e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107845015823841 }, "reported": { "count": 14, "observable_mse": 1.854493712016847e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107835702598095 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00660145605781308 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.022031523869373e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 14, "observable_mse": 3.0220360713428818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107840359210968 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0023105088575984434 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822486476390623e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.8198777474462986e-11, "numeric_tolerance_ratio": 3.811940354359698e-05 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676348190638237e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "reported": { "count": 84, "observable_mse": 1.8676328181754798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9971583853568556e-05 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.3699314371915534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360026644082518e-16 }, "reported": { "count": 14, "observable_mse": 3.369917976669967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360023997104558e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3460521586239338e-10, "numeric_tolerance_ratio": 0.0001341531308152326 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897396527579986e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00388339813798666 }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834072183817625 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.006540465866677444 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 6.000454959576018e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964682560414076 }, "reported": { "count": 72, "observable_mse": 6.000467237754492e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964771036058664 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006412915697540378 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.797699254297186e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038833962753415108 }, "reported": { "count": 14, "observable_mse": 7.79771198722301e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834111765027046 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.010733069131502397 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 1.6882830095710233e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043688067235052586 }, "reported": { "count": 12, "observable_mse": 1.6882742784218863e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004368808586150408 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.001296311477784105 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2916944026947021, "saved_at_worst": 1.2916944026947021, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.487018426439568, "worst_tolerance_index": [ 3093, 3 ], "actual_at_worst": 0.0007083266973495483, "saved_at_worst": 0.000708848237991333, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 1.531979432911612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007192505523562431 }, "reported": { "count": 6272, "observable_mse": 1.531979432911612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007192505057901144 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002708513307045594 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.2726763998216484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007062527816742659 }, "reported": { "count": 4092, "observable_mse": 1.272676763619529e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007062527816742659 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6333547209643545e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 1.6615829736110754e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007192506920546293 }, "reported": { "count": 896, "observable_mse": 1.6615824279142544e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007192506920546293 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.447916049061697e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 2.2679167159367353e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.007591899950057268 }, "reported": { "count": 1284, "observable_mse": 2.2679158064420335e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.007591899950057268 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.07436711711093e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2836270332336426, "saved_at_worst": 1.2836270332336426, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5258328609838328, "worst_tolerance_index": [ 1761, 6 ], "actual_at_worst": -0.0004856884479522705, "saved_at_worst": -0.00048513710498809814, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 2.6668303689802997e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013963649049401283 }, "reported": { "count": 3136, "observable_mse": 2.6668309146771207e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013963649049401283 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.442454105776573e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 2.4649105398566462e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013694006018340588 }, "reported": { "count": 1582, "observable_mse": 2.46491254074499e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013694005087018013 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0003930625367873124 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 2.6729636374511756e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013963649049401283 }, "reported": { "count": 448, "observable_mse": 2.672964910743758e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013963649049401283 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2698981891821986e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 2.9531676773331128e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.014340530149638653 }, "reported": { "count": 1106, "observable_mse": 2.9531669497373514e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.014340530149638653 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.254533764833367e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.284090280532837, "saved_at_worst": 1.284090280532837, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.416902777088917, "worst_tolerance_index": [ 645, 3 ], "actual_at_worst": 0.0010797232389450073, "saved_at_worst": 0.0010801851749420166, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 9.52184564084746e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026104753836989403 }, "reported": { "count": 3136, "observable_mse": 9.521851461613551e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026104751974344254 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0005159002755522424 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 8.958402759162709e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02523515187203884 }, "reported": { "count": 1001, "observable_mse": 8.958407124737278e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.025235150009393692 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0005286326718445574 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 9.404146840097383e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026104751974344254 }, "reported": { "count": 448, "observable_mse": 9.404151205671951e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026104751974344254 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.32490253115726e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 9.887426858767867e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02660731039941311 }, "reported": { "count": 1687, "observable_mse": 9.887434134725481e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02660731039941311 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.204721405824292e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2577426433563232, "saved_at_worst": 1.2577426433563232, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.3901293702431387, "worst_tolerance_index": [ 1178, 12 ], "actual_at_worst": 0.0006951391696929932, "saved_at_worst": 0.0006947219371795654, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00047963563702069223, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05295487120747566 }, "reported": { "count": 3136, "observable_mse": 0.00047963589895516634, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.052954867482185364 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0005917398364020188 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.00045876813237555325, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.050168175250291824 }, "reported": { "count": 407, "observable_mse": 0.0004587684525176883, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.050168171525001526 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006191463366830008 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00048218932352028787, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05295487120747566 }, "reported": { "count": 448, "observable_mse": 0.00048218961455859244, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05295487120747566 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002776502956625754 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.000482857518363744, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.053436823189258575 }, "reported": { "count": 2281, "observable_mse": 0.0004828577220905572, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.053436823189258575 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.0001943428200573798 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008089635521173477 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00808099564164877 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.650881858950015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.650880221859552e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6343922773059064e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008295925334095955 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6665429939166643e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6665442672092468e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2711741145488733e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008289745077490807 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5941945093800314e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5941997844493017e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.266673141096683e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.027659518644213676 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.701061935629696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.701080853119493e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8917489796876907e-10, "numeric_tolerance_ratio": 0.00018735732936814417 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025488151237368584 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.793230906827375e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.793223630869761e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.219692932613361e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025727983564138412 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.805288234725595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.805289689917117e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4424899806819776e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01637626811861992 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.188980554114096e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.188985647284426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0769799138515745e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 6.998432908560917e-16 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007534573320299387 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.9181770767318085e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.9181730749551207e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.001776687800884e-11, "numeric_tolerance_ratio": 3.9941152834059884e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0038363952189683914 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.709371286386158e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.709345820534509e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5443691858507307e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897401075053494e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-13, "numeric_tolerance_ratio": 4.5440247119973885e-07 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008749740198254585 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0526686057564802e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0526651496766135e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.4490002241757354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.611191146366764e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.080316883302202e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5991054169717245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5991059626685455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.448255872186213e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6771049558883533e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.677106411079876e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4527550979481874e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6066021998994984e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6065991076175123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.092281986027956e-11, "numeric_tolerance_ratio": 3.087321897422629e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.7853855752036907e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.7853810277301818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-11, "numeric_tolerance_ratio": 4.539369005564241e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 1.764368607837241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.7643691535340622e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4473570618693215e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 7.5585703598335385e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 7.558583092759363e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2723308806138392e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.499810635636095e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.4998324635089375e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.181151457204371e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6655649233143777e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6655654690111987e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.447894385869645e-06 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 1.8676328181754798e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 2.704847702261759e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 2.704847702261759e-06 }, "gradients": { "norms": { "prediction": 0.00013576024502981454, "weighted_latent": 2.6831793128829187e-18, "total": 0.00013576024502981454 }, "prediction_weighted_latent_dot": -9.274898775643064e-24, "prediction_weighted_latent_cosine": -0.025461669024432854, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "f27c7aa4ae44e1bd2421dfb90baa66d8366a995f651d65613378922892ca067a" } ], "final_losses": { "prediction_mse": 1.8676328181754798e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 2.704847702261759e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 2.704847702261759e-06 }, "training_seconds": 4.040001658722758e-05, "evaluation_seconds": 0.6968008000403643 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 1, "arm": "native", "status": "passed", "files": { "seed-1-native/result.json": "891d8fa190de61a1b640fb0ecf66d42cbd9686a51d3e0973c62483357ee349d6", "seed-1-native/checkpoint.json": "07f14c5b17dac49260c4caf900b11936e5242f4944ae0dbbd3a0ab1c060e6b5b", "inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "seed-1-native/diagnostics-arrays.npz": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b", "parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c", "parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.6193599700927734e-07, "max_tolerance_ratio": 0.45109829629777515, "worst_tolerance_index": [ 1, 6 ], "actual_at_worst": -0.00023978948593139648, "saved_at_worst": -0.00024025142192840576, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 1.8676346371648833e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 1.8676328181754798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.815598532142596e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.369931800989434e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 3.369917976669967e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.00013777889110753617 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.291385531425476, "saved_at_worst": 1.291385531425476, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.42046233387194104, "worst_tolerance_index": [ 87, 6 ], "actual_at_worst": 0.0009859204292297363, "saved_at_worst": 0.0009863823652267456, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.00011568346235435456, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04658252000808716 }, "reported": { "count": 784, "observable_mse": 0.00011568349145818502, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04658252000808716 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8770997512238653e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 0.0001269910135306418, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05011334642767906 }, "reported": { "count": 580, "observable_mse": 0.00012699104263447225, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05011334642767906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8738872518211027e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.00012861579307354987, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04658252000808716 }, "reported": { "count": 112, "observable_mse": 0.00012861573486588895, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04658252000808716 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.746852525276311e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 2.8653117624344304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "reported": { "count": 92, "observable_mse": 2.8653077606577426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004186348523944616 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.001776687800884e-11, "numeric_tolerance_ratio": 3.990343126672343e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.591147156432271e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205137968063354 }, "reported": { "count": 98, "observable_mse": 1.591144609847106e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120511934161186 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0013191059629536056 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2873621926701162e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120512865483761 }, "reported": { "count": 70, "observable_mse": 1.28735900943866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041205123998224735 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00032977647986314834 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.774524753272999e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120515659451485 }, "reported": { "count": 14, "observable_mse": 1.7745223885867745e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120510071516037 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.003957318410873048 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9266941055539064e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00412051472812891 }, "reported": { "count": 14, "observable_mse": 2.9266951969475485e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004120514262467623 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003297764363621434 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6111895092763007e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151817861944437 }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151880726218224 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004550374085107517 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.384819915983826e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815179457888007 }, "reported": { "count": 70, "observable_mse": 1.3848208254785277e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.0065727623458328764 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4024532902112696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151885382831097 }, "reported": { "count": 14, "observable_mse": 1.402454017807031e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003815184347331524 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0030335835414162663 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 2.9517736038542353e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151845801621675 }, "reported": { "count": 14, "observable_mse": 2.9517867005779408e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038151901680976152 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004044776351031832 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6099873391794972e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 98, "observable_mse": 1.609983155503869e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410440331324935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0029713781524049092 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2653170415433124e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104399122297764 }, "reported": { "count": 70, "observable_mse": 1.2653120393224526e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104403778910637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0033015311714488147 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.9126227925880812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "reported": { "count": 14, "observable_mse": 1.9126269762637094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004104407504200935 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.007923672718656958 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0307024644571356e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0041044000536203384 }, "reported": { "count": 14, "observable_mse": 3.0306960979942232e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00410439632833004 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0026412263323736225 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384033847600222 }, "reported": { "count": 98, "observable_mse": 1.5851730495342053e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384037107229233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022661433552030583 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2924613656650763e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374961368739605 }, "reported": { "count": 72, "observable_mse": 1.2924631846544798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004374966491013765 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.003563329461384987 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7241960449609905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043840352445840836 }, "reported": { "count": 14, "observable_mse": 1.724195317365229e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004384040366858244 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.00356108160832695 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.179249688400887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0044380780309438705 }, "reported": { "count": 12, "observable_mse": 3.179239502060227e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004438071511685848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004515324651932781 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5865916793700308e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355667158961296 }, "reported": { "count": 98, "observable_mse": 1.5865951354498975e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355666693300009 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0003243745464814044 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.2939838597958442e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043465327471494675 }, "reported": { "count": 72, "observable_mse": 1.2939861335325986e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004346531815826893 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006491621714371808 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.7255477359867655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355668555945158 }, "reported": { "count": 14, "observable_mse": 1.725551010167692e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004355671815574169 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0022706210151850026 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 3.180122803314589e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004410072695463896 }, "reported": { "count": 12, "observable_mse": 3.1801344448467717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00441007548943162 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001938898741991697 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6066007447079755e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 98, "observable_mse": 1.6065991076175123e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838496565819 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.122274160385132e-09, "numeric_tolerance_ratio": 0.0036308001127401722 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.2739343219436705e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843153178692 }, "reported": { "count": 70, "observable_mse": 1.2739328667521477e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107838962227106 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002970654539643297 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.8545022612670437e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107845015823841 }, "reported": { "count": 14, "observable_mse": 1.854493712016847e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107835702598095 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00660145605781308 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 3.022031523869373e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107843618839979 }, "reported": { "count": 14, "observable_mse": 3.0220360713428818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004107840359210968 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0023105088575984434 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0008219711598940194, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130469620227814 }, "reported": { "count": 98, "observable_mse": 0.0008219714509323239, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130469620227814 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00026893279647514115 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.0008061013650149107, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130469620227814 }, "reported": { "count": 84, "observable_mse": 0.0008061017724685371, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130471110343933 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0010545410041523351 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0009171902202069759, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130469620227814 }, "reported": { "count": 14, "observable_mse": 0.0009171901037916541, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.13130471110343933 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0010545410041523351 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897396527579986e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00388339813798666 }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834072183817625 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.080395102500916e-09, "numeric_tolerance_ratio": 0.006540465866677444 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 6.000454959576018e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964682560414076 }, "reported": { "count": 72, "observable_mse": 6.000467237754492e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037964771036058664 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006412915697540378 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 7.797699254297186e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038833962753415108 }, "reported": { "count": 14, "observable_mse": 7.79771198722301e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0038834111765027046 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.010733069131502397 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 1.6882830095710233e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0043688067235052586 }, "reported": { "count": 12, "observable_mse": 1.6882742784218863e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.004368808586150408 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.001296311477784105 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2916944026947021, "saved_at_worst": 1.2916944026947021, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.487018426439568, "worst_tolerance_index": [ 3093, 3 ], "actual_at_worst": 0.0007083266973495483, "saved_at_worst": 0.000708848237991333, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.00041410542326048017, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0698724240064621 }, "reported": { "count": 6272, "observable_mse": 0.00041410542326048017, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0698724240064621 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.00048100444837473333, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07464633882045746 }, "reported": { "count": 4092, "observable_mse": 0.00048100450658239424, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07464633882045746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.553633802648516e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.0004096943885087967, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0698724240064621 }, "reported": { "count": 896, "observable_mse": 0.00040969435940496624, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0698724240064621 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.79583909497198e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.00020398187916725874, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0518018938601017 }, "reported": { "count": 1284, "observable_mse": 0.00020398193737491965, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.0518018938601017 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.704406502354309e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.1448103189468384, "saved_at_worst": 1.1448103189468384, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5258328609838328, "worst_tolerance_index": [ 1761, 6 ], "actual_at_worst": -0.0004856884479522705, "saved_at_worst": -0.00048513710498809814, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0008384898537769914, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09975315630435944 }, "reported": { "count": 3136, "observable_mse": 0.0008384898537769914, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09975315630435944 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.0009020490688271821, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10331261903047562 }, "reported": { "count": 1582, "observable_mse": 0.0009020491852425039, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10331261903047562 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.0001067829724933912 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0008328369585797191, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09975315630435944 }, "reported": { "count": 448, "observable_mse": 0.0008328371332027018, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09975315630435944 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016119782896502883 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.0007498656050302088, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09442884474992752 }, "reported": { "count": 1106, "observable_mse": 0.0007498656632378697, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09442883729934692 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0007134600738267936 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2048335075378418, "saved_at_worst": 1.2048335075378418, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.513429641723633e-07, "max_tolerance_ratio": 0.416902777088917, "worst_tolerance_index": [ 645, 3 ], "actual_at_worst": 0.0010797232389450073, "saved_at_worst": 0.0010801851749420166, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.006654600612819195, "exact_state_accuracy": 0.9754464285714286, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.9754464285714286 }, "normalized_latent_rmse": 0.2049500048160553 }, "reported": { "count": 3136, "observable_mse": 0.006654600612819195, "exact_state_accuracy": 0.9754464030265808, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.9754464030265808 }, "normalized_latent_rmse": 0.2049500048160553 }, "differences": [], "max_errors": { "accuracy_abs": 2.5544847792602354e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00838522706180811, "exact_state_accuracy": 0.964035964035964, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.964035964035964 }, "normalized_latent_rmse": 0.22751471400260925 }, "reported": { "count": 1001, "observable_mse": 0.00838522519916296, "exact_state_accuracy": 0.9640359282493591, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.9640359282493591 }, "normalized_latent_rmse": 0.22751472890377045 }, "differences": [], "max_errors": { "accuracy_abs": 3.578660490166641e-08, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0010131206602330654 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00667498679831624, "exact_state_accuracy": 0.9754464285714286, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.9754464285714286 }, "normalized_latent_rmse": 0.2049499899148941 }, "reported": { "count": 448, "observable_mse": 0.006674987263977528, "exact_state_accuracy": 0.9754464626312256, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.9754464626312256 }, "normalized_latent_rmse": 0.2049499899148941 }, "differences": [], "max_errors": { "accuracy_abs": 3.405979698278827e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002792573571037702 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.005622301250696182, "exact_state_accuracy": 0.98221695317131, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.98221695317131 }, "normalized_latent_rmse": 0.1903003305196762 }, "reported": { "count": 1687, "observable_mse": 0.005622302182018757, "exact_state_accuracy": 0.982217013835907, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 0.982217013835907 }, "normalized_latent_rmse": 0.1903003305196762 }, "differences": [], "max_errors": { "accuracy_abs": 6.066459701248306e-08, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005961493791148332 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.1234519481658936, "saved_at_worst": 1.1234519481658936, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.289912223815918e-07, "max_tolerance_ratio": 0.3310985707951908, "worst_tolerance_index": [ 1621, 9 ], "actual_at_worst": -0.00170096755027771, "saved_at_worst": -0.00170135498046875, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.01799476332962513, "exact_state_accuracy": 0.9585459183673469, "per_field_accuracy": { "x": 1.0, "y": 0.9862882653061225, "h": 0.9674744897959183 }, "normalized_latent_rmse": 0.2951428294181824 }, "reported": { "count": 3136, "observable_mse": 0.01799476519227028, "exact_state_accuracy": 0.9585459232330322, "per_field_accuracy": { "x": 1.0, "y": 0.9862882494926453, "h": 0.9674744606018066 }, "normalized_latent_rmse": 0.29514279961586 }, "differences": [], "max_errors": { "accuracy_abs": 2.9194111683672475e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0009766680526367667 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.02545979805290699, "exact_state_accuracy": 0.9238329238329238, "per_field_accuracy": { "x": 1.0, "y": 0.9901719901719902, "h": 0.9336609336609336 }, "normalized_latent_rmse": 0.33765652775764465 }, "reported": { "count": 407, "observable_mse": 0.02545979991555214, "exact_state_accuracy": 0.923832893371582, "per_field_accuracy": { "x": 1.0, "y": 0.9901719689369202, "h": 0.9336609244346619 }, "normalized_latent_rmse": 0.33765649795532227 }, "differences": [], "max_errors": { "accuracy_abs": 3.0461341782306306e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0008572347291931027 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.018050488084554672, "exact_state_accuracy": 0.9620535714285714, "per_field_accuracy": { "x": 1.0, "y": 0.9866071428571429, "h": 0.96875 }, "normalized_latent_rmse": 0.2951428294181824 }, "reported": { "count": 448, "observable_mse": 0.01805048994719982, "exact_state_accuracy": 0.9620535969734192, "per_field_accuracy": { "x": 1.0, "y": 0.9866071939468384, "h": 0.9687500596046448 }, "normalized_latent_rmse": 0.2951428294181824 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006640330178678031 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.016651831567287445, "exact_state_accuracy": 0.964050854888207, "per_field_accuracy": { "x": 1.0, "y": 0.9855326611135466, "h": 0.9732573432704954 }, "normalized_latent_rmse": 0.2868955135345459 }, "reported": { "count": 2281, "observable_mse": 0.016651831567287445, "exact_state_accuracy": 0.9640508890151978, "per_field_accuracy": { "x": 1.0, "y": 0.9855326414108276, "h": 0.9732573628425598 }, "normalized_latent_rmse": 0.2868955135345459 }, "differences": [], "max_errors": { "accuracy_abs": 3.4126990788863054e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008089635521173477 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6376299754483625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00808099564164877 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.650881858950015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.650880221859552e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6343922773059064e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008295925334095955 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6665429939166643e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6665442672092468e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2711741145488733e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008289745077490807 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5941945093800314e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5941997844493017e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.266673141096683e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.027659518644213676 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.701061935629696e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.701080853119493e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8917489796876907e-10, "numeric_tolerance_ratio": 0.00018735732936814417 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025488151237368584 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.793230906827375e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.793223630869761e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.219692932613361e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.025727983564138412 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.805288234725595e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.805289689917117e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4424899806819776e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.01637626811861992 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.188980554114096e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.188985647284426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0769799138515745e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.2242463231086731 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.008719741366803646, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.008719741366803646, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.22663171589374542 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.009586147964000702, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.009586147964000702, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0822481019422412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0822448277613148e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.267377446594027e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0038363952189683914 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 8.709371286386158e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 8.709345820534509e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5443691858507307e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 7.5897401075053494e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.5897405622527e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-13, "numeric_tolerance_ratio": 4.5440247119973885e-07 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008749740198254585 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.0526686057564802e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0526651496766135e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.4490002241757354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.611191146366764e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6111920558614656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.080316883302202e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5991054169717245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5991059626685455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.448255872186213e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6771049558883533e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.677106411079876e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4527550979481874e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.001706356881186366, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0017063572304323316, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.0002983387219492074 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0008994330419227481, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0008994329255074263, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010680860428480968 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 1.764368607837241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.7643691535340622e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4473570618693215e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 7.5585703598335385e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 7.558583092759363e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2723308806138392e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.499810635636095e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.4998324635089375e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.181151457204371e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.6655649233143777e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.6655654690111987e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.447894385869645e-06 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 70, "error_count": 28, "per_field_correct_counts": { "x": 98, "y": 98, "h": 70 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 177, "error_count": 19, "per_field_correct_counts": { "x": 196, "y": 196, "h": 177 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 58, "error_count": 40, "per_field_correct_counts": { "x": 98, "y": 62, "h": 86 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 55, "error_count": 43, "per_field_correct_counts": { "x": 98, "y": 91, "h": 55 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3006, "accuracy": 0.9585459183673469 }, "numpy": { "cases": 3136, "correct": 3006, "accuracy": 0.9585459183673469 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0008061017724685371, "latent_mse": 0.01724092662334442, "scaled_prediction": 0.00011674577399389818, "scaled_weighted_latent": 0.000249696196988225, "total": 0.00036644196370616555 }, "gradients": { "norms": { "prediction": 0.0013187306467443705, "weighted_latent": 0.0013792344834655523, "total": 0.0023915665224194527 }, "prediction_weighted_latent_dot": 1.0391254363639746e-06, "prediction_weighted_latent_cosine": 0.571312557453286, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "07f14c5b17dac49260c4caf900b11936e5242f4944ae0dbbd3a0ab1c060e6b5b" } ], "final_losses": { "prediction_mse": 0.0008061017724685371, "latent_mse": 0.01724092662334442, "scaled_prediction": 0.00011674577399389818, "scaled_weighted_latent": 0.000249696196988225, "total": 0.00036644196370616555 }, "training_seconds": 3.9599952287971973e-05, "evaluation_seconds": 0.53689819999272 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 2, "arm": "latent_only", "status": "passed", "files": { "seed-2-latent_only/result.json": "ff7cdb5c62fd466904afbdbd3a4e9642acd8ccdffccc3e52e4a1092c51bda229", "seed-2-latent_only/checkpoint.json": "98a0907e90e006e7eceee8a2a878b8fafbfaa046337219367a17ac48357802cd", "inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "seed-2-latent_only/diagnostics-arrays.npz": "f750723b6f773a15bc25f177a7aff420f0aba851bd55805f2ab9a5f07314e702" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.172325134277344e-07, "max_tolerance_ratio": 0.38184041187270573, "worst_tolerance_index": [ 90, 15 ], "actual_at_worst": -0.0009272992610931396, "saved_at_worst": -0.0009268820285797119, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.4284559913212433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.428454718028661e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2702079400172872e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926299571990967, "saved_at_worst": 1.2926299571990967, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5302791797917392, "worst_tolerance_index": [ 545, 1 ], "actual_at_worst": -0.0012408345937728882, "saved_at_worst": -0.0012402385473251343, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.365978252782952e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033088121563196182 }, "reported": { "count": 784, "observable_mse": 2.36597716138931e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033088119234889746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017494472458728073 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.978542422875762e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003269587177783251 }, "reported": { "count": 580, "observable_mse": 1.978542968572583e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003269587177783251 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4461926843963654e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.7823572963825427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003308812854811549 }, "reported": { "count": 112, "observable_mse": 2.782348747132346e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003308812854811549 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.549250196665525e-11, "numeric_tolerance_ratio": 8.525529201174e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 4.30161198892165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "reported": { "count": 92, "observable_mse": 4.301610169932246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.811198334370962e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4489330826327205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 98, "observable_mse": 2.448930717946496e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749081823974848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333975094181 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0924091586493887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925644695759 }, "reported": { "count": 70, "observable_mse": 2.0924004275002517e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749084152281284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333754161296 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1905292669543996e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 14, "observable_mse": 2.1905329049332067e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474907483905554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.792795956134796e-08, "numeric_tolerance_ratio": 0.01330469955564529 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.489958519116044e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474924247711897 }, "reported": { "count": 14, "observable_mse": 4.489981074584648e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00347490725107491 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6996636986732483e-08, "numeric_tolerance_ratio": 0.012613546549922739 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4650891646160744e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355574123561382 }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017459385640157722 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0494757336564362e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 70, "observable_mse": 2.049479189736303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355571795254946 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00034918772499636073 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.5227938749594614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335555549710989 }, "reported": { "count": 14, "observable_mse": 2.5227842343156226e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355604391545057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.0036664702162613125 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.485453246161342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 14, "observable_mse": 4.485439421841875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355567138642073 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006983754743791358 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4724424292799085e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488371381536126 }, "reported": { "count": 98, "observable_mse": 2.4724389732000418e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883692860603333 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0954757928848267e-09, "numeric_tolerance_ratio": 0.0015535427214692392 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9836363208014518e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488370683044195 }, "reported": { "count": 70, "observable_mse": 1.983637412195094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012083110264445502 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.7921978471567854e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883730113506317 }, "reported": { "count": 14, "observable_mse": 2.792183840938378e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488372080028057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006904632887422107 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.596718281391077e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883723128587008 }, "reported": { "count": 14, "observable_mse": 4.596703365677968e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0024166220528891004 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4542685423512012e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660863498225808 }, "reported": { "count": 98, "observable_mse": 2.4542630853829905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660864196717739 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0005113087436513964 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9375931515241973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036534646060317755 }, "reported": { "count": 72, "observable_mse": 1.9375880583538674e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00365346553735435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006821144214759868 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0936709663365036e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608625669032335 }, "reported": { "count": 14, "observable_mse": 3.093665145570412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608658265322447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002386107185697774 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8083511501317844e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704948117956519 }, "reported": { "count": 12, "observable_mse": 4.808346784557216e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704944159835577 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.958120942115784e-09, "numeric_tolerance_ratio": 0.002888097095437761 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.454017158015631e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00366382603533566 }, "reported": { "count": 98, "observable_mse": 2.4540126105421223e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638197489082813 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004600783305236997 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9373417671886273e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656336572021246 }, "reported": { "count": 72, "observable_mse": 1.937336492119357e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656331915408373 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0034098562497762368 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.09398710669484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638290621340275 }, "reported": { "count": 14, "observable_mse": 3.0939725547796115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003663820680230856 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.0061343773221983556 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8074376536533237e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003708441974595189 }, "reported": { "count": 12, "observable_mse": 4.8074492951855063e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00370843056589365 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1408701539039612e-08, "numeric_tolerance_ratio": 0.008322398019379602 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4505963665433228e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 98, "observable_mse": 2.4505910914740525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0662389943026938e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 70, "observable_mse": 2.066234446829185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.32611782848835e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416656516492367 }, "reported": { "count": 14, "observable_mse": 2.3261030946741812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416670486330986 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.010412299112111879 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.4968615839025006e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416659077629447 }, "reported": { "count": 14, "observable_mse": 4.4968630390940234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416667925193906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006594457363167671 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7606145522440784e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.6749262360681314e-07 }, "reported": { "count": 98, "observable_mse": 2.760614916041959e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.663332452357281e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1593783710850403e-09, "numeric_tolerance_ratio": 0.001159347493807002 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.428484003758058e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.6760042715068266e-07 }, "reported": { "count": 84, "observable_mse": 2.428484003758058e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.656196045336401e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.980822617042577e-09, "numeric_tolerance_ratio": 0.001980770003908066 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.7533987526549026e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.668449496923131e-07 }, "reported": { "count": 14, "observable_mse": 4.75340239063371e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.7057549800701963e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.730548314706539e-09, "numeric_tolerance_ratio": 0.003730447377940832 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218651813280303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703649388626218 }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703637048602104 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.009004926261465663 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.265882656298345e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003696256782859564 }, "reported": { "count": 72, "observable_mse": 1.2658831110456958e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036962456069886684 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.008159806136714668 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4861635463603307e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703654045239091 }, "reported": { "count": 14, "observable_mse": 1.4861591807857621e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037036393769085407 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4668330550193787e-08, "numeric_tolerance_ratio": 0.010703967133658529 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.2827452994533814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037476930301636457 }, "reported": { "count": 12, "observable_mse": 2.2827287466498092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00374767673201859 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-08, "numeric_tolerance_ratio": 0.01185519951732077 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29456627368927, "saved_at_worst": 1.29456627368927, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.48837328246516154, "worst_tolerance_index": [ 1481, 14 ], "actual_at_worst": 6.841123104095459e-05, "saved_at_worst": 6.890296936035156e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.3900793166831136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006172883789986372 }, "reported": { "count": 6272, "observable_mse": 2.3900787709862925e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006172883789986372 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.44395672523841e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 2.0221228623995557e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006107353139668703 }, "reported": { "count": 4092, "observable_mse": 2.0221217710059136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006107353139668703 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0891911649600609e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.621443672978785e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006172882858663797 }, "reported": { "count": 896, "observable_mse": 2.6214440367766656e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006172883324325085 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.000287927191441092 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.401274807401933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006377234123647213 }, "reported": { "count": 1284, "observable_mse": 3.401275898795575e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.006377233657985926 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028433452012249446 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2953126430511475, "saved_at_worst": 1.2953126430511475, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4286914392087942, "worst_tolerance_index": [ 2524, 1 ], "actual_at_worst": 0.0011226087808609009, "saved_at_worst": 0.001123085618019104, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.592955908970907e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011947340331971645 }, "reported": { "count": 3136, "observable_mse": 3.592955908970907e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011947340331971645 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 3.2867963454918936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011877006851136684 }, "reported": { "count": 1582, "observable_mse": 3.286795981694013e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01187700591981411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004257084255629218 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.645531978690997e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01194733940064907 }, "reported": { "count": 448, "observable_mse": 3.645533070084639e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011947340331971645 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00042434416222123303 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 4.0095834265230224e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012047233060002327 }, "reported": { "count": 1106, "observable_mse": 4.009585245512426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012047232128679752 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00042242153989206836 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2925742864608765, "saved_at_worst": 1.2925742864608765, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4916956737725486, "worst_tolerance_index": [ 36, 11 ], "actual_at_worst": 0.0009863413870334625, "saved_at_worst": 0.0009858012199401855, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00010225340520264581, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022724181413650513 }, "reported": { "count": 3136, "observable_mse": 0.00010225344158243388, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022724179551005363 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0005691953701475557 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 9.7694355645217e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0224163755774498 }, "reported": { "count": 1001, "observable_mse": 9.76944065769203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0224163755774498 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.04389430384333e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00010278232366545126, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02272418513894081 }, "reported": { "count": 448, "observable_mse": 0.00010278236004523933, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02272418513894081 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.600967216199065e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00010481807839823887, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022904863581061363 }, "reported": { "count": 1687, "observable_mse": 0.0001048181438818574, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022904863581061363 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.548361852765083e-11, "numeric_tolerance_ratio": 6.48043513453026e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2814282178878784, "saved_at_worst": 1.2814282178878784, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 4.917383193969727e-07, "max_tolerance_ratio": 0.417693559955126, "worst_tolerance_index": [ 2629, 11 ], "actual_at_worst": 0.0007911957800388336, "saved_at_worst": 0.0007916465401649475, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00046192927402444184, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04700247570872307 }, "reported": { "count": 3136, "observable_mse": 0.0004619293613359332, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04700247570872307 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.345639542633262e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.00044614719809032977, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04471798986196518 }, "reported": { "count": 407, "observable_mse": 0.0004461473145056516, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04471799358725548 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006808163191366691 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0004608675662893802, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04700247570872307 }, "reported": { "count": 448, "observable_mse": 0.0004608677118085325, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04700247570872307 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.000139108108708231 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0004649537440855056, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04739852622151375 }, "reported": { "count": 2281, "observable_mse": 0.00046495391870848835, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04739852622151375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016686455009440977 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006795178167521954 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4145125280483626e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4145105271600187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9960688147343306e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006793184205889702 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4182987544918433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4183040295611136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.262343324217096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007076622452586889 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4742850655457005e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.474279244779609e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.806399437731063e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00707575399428606 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4824970751069486e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4825025320751593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.44345482026303e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02292351797223091 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.174026930239052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.173992005642503e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00034710295461388345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02197711542248726 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.503961438080296e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.5039843573467806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.2919266484677792e-10, "numeric_tolerance_ratio": 0.0002279380971257544 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.022106481716036797 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.032768189674243e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.03276673448272e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4450289711579617e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.013866852968931198 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.329649189254269e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.3296459150733426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.26331524211703e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 5.192282515054103e-07 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7606429284787737e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.760647657851223e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.716352262392452e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0063375686295330524 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8662645490840077e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8662767363130115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2187229003757238e-10, "numeric_tolerance_ratio": 0.00012152396871314546 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.003950412385165691 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.004109410336241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0041121388203464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-11, "numeric_tolerance_ratio": 2.7230268541460572e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218641808838584e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4531253780259127e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007395219057798386 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7916874387301505e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7916930775973015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.638867150992155e-11, "numeric_tolerance_ratio": 5.623168989051261e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.465091529302299e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.266129628595387e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4603585188742727e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.460366340528708e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.821654435247183e-11, "numeric_tolerance_ratio": 7.802457531363612e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.408349246252328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4083446987788193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-11, "numeric_tolerance_ratio": 4.536547937688154e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.450601641612593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4506001864210702e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4516341479231694e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8193089747219346e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8193100661155768e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883253156100062e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.6724419512902386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.672434493433684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.457856554538012e-11, "numeric_tolerance_ratio": 7.437979042782642e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.5223119589791168e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.5223105947370641e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3642420526593924e-11, "numeric_tolerance_ratio": 1.3621684092581624e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5361989426310174e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5361960322479717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.90591898445941e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.4043375964974985e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.404349238029681e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011590483644883742 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0006819527479819953, "latent_mse": 0.01885349676012993, "scaled_prediction": 9.876557305688038e-05, "scaled_weighted_latent": 0.0002730506530497223, "total": 0.0002730506530497223 }, "gradients": { "norms": { "prediction": 0.0009655424510128796, "weighted_latent": 0.001503756851889193, "total": 0.001503756851889193 }, "prediction_weighted_latent_dot": 9.367491315970256e-07, "prediction_weighted_latent_cosine": 0.6451702116278689, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0007797321886755526, "latent_mse": 0.017864245921373367, "scaled_prediction": 0.00011292673298157752, "scaled_weighted_latent": 0.00025872356491163373, "total": 0.00025872356491163373 }, "gradients": { "norms": { "prediction": 0.0017306143417954445, "weighted_latent": 0.0014648055657744408, "total": 0.0014648055657744408 }, "prediction_weighted_latent_dot": 1.1071567769249668e-06, "prediction_weighted_latent_cosine": 0.43674590614962144, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.03140963986515999 }, { "step": 100, "losses": { "prediction_mse": 0.00012653719750232995, "latent_mse": 0.00035402210778556764, "scaled_prediction": 1.8326078134123236e-05, "scaled_weighted_latent": 5.127216809341917e-06, "total": 5.127216809341917e-06 }, "gradients": { "norms": { "prediction": 0.0017391019500792027, "weighted_latent": 0.00022246266598813236, "total": 0.00022246266598813236 }, "prediction_weighted_latent_dot": 1.5913174422621523e-07, "prediction_weighted_latent_cosine": 0.41131509061071686, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.001169891795143485 }, { "step": 200, "losses": { "prediction_mse": 2.367920933465939e-05, "latent_mse": 3.444951914843841e-07, "scaled_prediction": 3.4294027955183992e-06, "scaled_weighted_latent": 4.989241020325608e-09, "total": 4.989241020325608e-09 }, "gradients": { "norms": { "prediction": 7.438002648996189e-05, "weighted_latent": 6.940542789379833e-06, "total": 6.940542789379833e-06 }, "prediction_weighted_latent_dot": -3.408873461285822e-10, "prediction_weighted_latent_cosine": -0.6603301324194677, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 6.886219489388168e-05 }, { "step": 300, "losses": { "prediction_mse": 2.4286680854856968e-05, "latent_mse": 2.895395728730321e-12, "scaled_prediction": 3.5173814012523508e-06, "scaled_weighted_latent": 4.193331811885685e-14, "total": 4.193331811885685e-14 }, "gradients": { "norms": { "prediction": 0.00010050291166407987, "weighted_latent": 2.00909866521215e-08, "total": 2.00909866521215e-08 }, "prediction_weighted_latent_dot": 1.4684928620334325e-12, "prediction_weighted_latent_cosine": 0.7272637331027412, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.0570345665428249e-07 }, { "step": 400, "losses": { "prediction_mse": 2.4284863684442826e-05, "latent_mse": 7.688360852912274e-14, "scaled_prediction": 3.517118329909863e-06, "scaled_weighted_latent": 1.1134867484500458e-15, "total": 1.1134867484500458e-15 }, "gradients": { "norms": { "prediction": 0.00010040963388746604, "weighted_latent": 3.113640012486485e-09, "total": 3.113640012486485e-09 }, "prediction_weighted_latent_dot": 2.099499637727223e-13, "prediction_weighted_latent_cosine": 0.6715402079953351, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.5189179691788013e-07 }, { "step": 500, "losses": { "prediction_mse": 2.428483458061237e-05, "latent_mse": 6.981860158376471e-14, "scaled_prediction": 3.5171140098100295e-06, "scaled_weighted_latent": 1.0111660154546735e-15, "total": 1.0111660154546735e-15 }, "gradients": { "norms": { "prediction": 0.00010040985944215208, "weighted_latent": 3.031748407877899e-09, "total": 3.031748407877899e-09 }, "prediction_weighted_latent_dot": 2.0976327771114744e-13, "prediction_weighted_latent_cosine": 0.6890646067087806, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.7064159862911765e-07 }, { "step": 600, "losses": { "prediction_mse": 2.4284812752739526e-05, "latent_mse": 6.92448350178218e-14, "scaled_prediction": 3.5171108265785733e-06, "scaled_weighted_latent": 1.0028563046047536e-15, "total": 1.0028563046047536e-15 }, "gradients": { "norms": { "prediction": 0.00010040901543106884, "weighted_latent": 3.0500808545497193e-09, "total": 3.0500808545497193e-09 }, "prediction_weighted_latent_dot": 2.077947866942556e-13, "prediction_weighted_latent_cosine": 0.6785011477866526, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.9513116455982527e-07 }, { "step": 700, "losses": { "prediction_mse": 2.4284869141411036e-05, "latent_mse": 7.430133033359612e-14, "scaled_prediction": 3.5171190120308893e-06, "scaled_weighted_latent": 1.0760882321255187e-15, "total": 1.0760882321255187e-15 }, "gradients": { "norms": { "prediction": 0.00010041041969088838, "weighted_latent": 3.114990265729034e-09, "total": 3.114990265729034e-09 }, "prediction_weighted_latent_dot": 2.1831597944374503e-13, "prediction_weighted_latent_cosine": 0.6979913627514204, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.2110079928315827e-07 }, { "step": 800, "losses": { "prediction_mse": 2.4284858227474615e-05, "latent_mse": 7.316722775612197e-14, "scaled_prediction": 3.517117420415161e-06, "scaled_weighted_latent": 1.0596633103661921e-15, "total": 1.0596633103661921e-15 }, "gradients": { "norms": { "prediction": 0.00010040940105682239, "weighted_latent": 3.103554968575395e-09, "total": 3.103554968575395e-09 }, "prediction_weighted_latent_dot": 2.1546756349619112e-13, "prediction_weighted_latent_cosine": 0.6914297826108758, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.905540270603524e-07 }, { "step": 900, "losses": { "prediction_mse": 2.4284858227474615e-05, "latent_mse": 7.161070645972031e-14, "scaled_prediction": 3.517117420415161e-06, "scaled_weighted_latent": 1.037120587266203e-15, "total": 1.037120587266203e-15 }, "gradients": { "norms": { "prediction": 0.00010040773486252874, "weighted_latent": 3.1429283620099113e-09, "total": 3.1429283620099113e-09 }, "prediction_weighted_latent_dot": 1.9667293238868205e-13, "prediction_weighted_latent_cosine": 0.6232222376147312, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.7591416912619025e-07 }, { "step": 1000, "losses": { "prediction_mse": 2.428484003758058e-05, "latent_mse": 7.05537787481364e-14, "scaled_prediction": 3.5171149193047313e-06, "scaled_weighted_latent": 1.0218133254807785e-15, "total": 1.0218133254807785e-15 }, "gradients": { "norms": { "prediction": 0.00010040798224508762, "weighted_latent": 3.095038891842705e-09, "total": 3.095038891842705e-09 }, "prediction_weighted_latent_dot": 2.1120832947169044e-13, "prediction_weighted_latent_cosine": 0.6796364944217402, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.087874287961313e-07 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "aa58dbb3d60e93a7ba96625c370c99c56f57dd6b174a7e40abad84e98e9cac39" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "7efd99d08941e8b46c20b7d4b7394b1024c87e7f310403a25de38d6795156ff5" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "2ae06d59e09d369623d533ad194c9d63fe8bf0a4b141f7950b0faedb58402076" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "aa9e656b7f28eef673d21d5012f224692c51cbe3b8d670cd17e29f5bab4785f1" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "01b2901ad9c01862b2a992897fd6c3001d6f8f1acb4236e79604e870bc0ede41" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "733983eb538fef59a951b02b83d9c228b3cd8533a7ec8123a8ba1b4eaa5a6c95" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "30ff275a169fa2a6ae7f10ca9cf651bc0ae107e456350f4113c60826618f0531" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "f947f2ca56f1ae84ba18478db8296a21837fc5cfa17f382023155b6fbf5b4707" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "29750935156390726ab063c1f8e3ec7f83a25f0f6c6eca066f62b17a260af58e" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "69243f60cb26df352c8e97589ed6e917f9ce98b40b8b7ec29889763d66ed5b64" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "e7ab4b9fcee645d495ed002c4e7bbc41e216974e8ada148404b161dd28dbb827" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "98a0907e90e006e7eceee8a2a878b8fafbfaa046337219367a17ac48357802cd" } ], "final_losses": { "prediction_mse": 2.428484003758058e-05, "latent_mse": 7.05537787481364e-14, "scaled_prediction": 3.5171149193047313e-06, "scaled_weighted_latent": 1.0218133254807785e-15, "total": 1.0218133254807785e-15 }, "training_seconds": 2.756893600046169, "evaluation_seconds": 0.5482100999797694 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 2, "arm": "refit", "status": "passed", "files": { "seed-2-refit/result.json": "c9ba2578682b8785b6a32a4b326b907148f9c1c11ea4fbfc188cf90a838fd908", "seed-2-refit/checkpoint.json": "bda2f8457e2047e74b2f4197dcf8e137d5b22a58fb906b015a5af31e57ab3bd4", "inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "seed-2-refit/diagnostics-arrays.npz": "a1f9bbdf9024c8215f4cb7b88b9dc9ad2907426b95ab3e1cf1fbc0609c935279" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.172325134277344e-07, "max_tolerance_ratio": 0.38184041187270573, "worst_tolerance_index": [ 90, 15 ], "actual_at_worst": -0.0009272992610931396, "saved_at_worst": -0.0009268820285797119, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.4284559913212433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.428454718028661e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2702079400172872e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926299571990967, "saved_at_worst": 1.2926299571990967, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5302791797917392, "worst_tolerance_index": [ 545, 1 ], "actual_at_worst": -0.0012408345937728882, "saved_at_worst": -0.0012402385473251343, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.365975160500966e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033088121563196182 }, "reported": { "count": 784, "observable_mse": 2.3659733415115625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033088119234889746 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017494472458728073 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.9785386029980145e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003269587177783251 }, "reported": { "count": 580, "observable_mse": 1.9785384210990742e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003269587177783251 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-12, "numeric_tolerance_ratio": 1.8153975697046262e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.7823556592920795e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003308812854811549 }, "reported": { "count": 112, "observable_mse": 2.7823460186482407e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003308812854811549 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.640643838793039e-11, "numeric_tolerance_ratio": 9.613894657269682e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 4.30161198892165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "reported": { "count": 92, "observable_mse": 4.301610169932246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.811198334370962e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4489330826327205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 98, "observable_mse": 2.448930717946496e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749081823974848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333975094181 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0924091586493887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925644695759 }, "reported": { "count": 70, "observable_mse": 2.0924004275002517e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749084152281284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333754161296 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1905292669543996e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 14, "observable_mse": 2.1905329049332067e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474907483905554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.792795956134796e-08, "numeric_tolerance_ratio": 0.01330469955564529 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.489958519116044e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474924247711897 }, "reported": { "count": 14, "observable_mse": 4.489981074584648e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00347490725107491 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6996636986732483e-08, "numeric_tolerance_ratio": 0.012613546549922739 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4650891646160744e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355574123561382 }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017459385640157722 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0494757336564362e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 70, "observable_mse": 2.049479189736303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355571795254946 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00034918772499636073 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.5227938749594614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335555549710989 }, "reported": { "count": 14, "observable_mse": 2.5227842343156226e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355604391545057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.0036664702162613125 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.485453246161342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 14, "observable_mse": 4.485439421841875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355567138642073 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006983754743791358 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4724424292799085e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488371381536126 }, "reported": { "count": 98, "observable_mse": 2.4724389732000418e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883692860603333 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0954757928848267e-09, "numeric_tolerance_ratio": 0.0015535427214692392 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9836363208014518e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488370683044195 }, "reported": { "count": 70, "observable_mse": 1.983637412195094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012083110264445502 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.7921978471567854e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883730113506317 }, "reported": { "count": 14, "observable_mse": 2.792183840938378e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488372080028057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006904632887422107 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.596718281391077e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883723128587008 }, "reported": { "count": 14, "observable_mse": 4.596703365677968e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0024166220528891004 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4542685423512012e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660863498225808 }, "reported": { "count": 98, "observable_mse": 2.4542630853829905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660864196717739 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0005113087436513964 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9375931515241973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036534646060317755 }, "reported": { "count": 72, "observable_mse": 1.9375880583538674e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00365346553735435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006821144214759868 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0936709663365036e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608625669032335 }, "reported": { "count": 14, "observable_mse": 3.093665145570412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608658265322447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002386107185697774 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8083511501317844e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704948117956519 }, "reported": { "count": 12, "observable_mse": 4.808346784557216e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704944159835577 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.958120942115784e-09, "numeric_tolerance_ratio": 0.002888097095437761 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.454017158015631e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00366382603533566 }, "reported": { "count": 98, "observable_mse": 2.4540126105421223e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638197489082813 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004600783305236997 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9373417671886273e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656336572021246 }, "reported": { "count": 72, "observable_mse": 1.937336492119357e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656331915408373 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0034098562497762368 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.09398710669484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638290621340275 }, "reported": { "count": 14, "observable_mse": 3.0939725547796115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003663820680230856 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.0061343773221983556 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8074376536533237e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003708441974595189 }, "reported": { "count": 12, "observable_mse": 4.8074492951855063e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00370843056589365 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1408701539039612e-08, "numeric_tolerance_ratio": 0.008322398019379602 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4505963665433228e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 98, "observable_mse": 2.4505910914740525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0662389943026938e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 70, "observable_mse": 2.066234446829185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.32611782848835e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416656516492367 }, "reported": { "count": 14, "observable_mse": 2.3261030946741812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416670486330986 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.010412299112111879 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.4968615839025006e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416659077629447 }, "reported": { "count": 14, "observable_mse": 4.4968630390940234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416667925193906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006594457363167671 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605889044934884e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.632583573764565e-05 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.428456355119124e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "reported": { "count": 84, "observable_mse": 2.428454718028661e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6331244943079406e-05 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.7533838369417936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360026644082518e-16 }, "reported": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360023997104558e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6207679087541517e-06 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218651813280303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703649388626218 }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703637048602104 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.009004926261465663 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.265882656298345e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003696256782859564 }, "reported": { "count": 72, "observable_mse": 1.2658831110456958e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036962456069886684 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.008159806136714668 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4861635463603307e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703654045239091 }, "reported": { "count": 14, "observable_mse": 1.4861591807857621e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037036393769085407 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4668330550193787e-08, "numeric_tolerance_ratio": 0.010703967133658529 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.2827452994533814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037476930301636457 }, "reported": { "count": 12, "observable_mse": 2.2827287466498092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00374767673201859 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-08, "numeric_tolerance_ratio": 0.01185519951732077 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29456627368927, "saved_at_worst": 1.29456627368927, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.48837328246516154, "worst_tolerance_index": [ 1481, 14 ], "actual_at_worst": 6.841123104095459e-05, "saved_at_worst": 6.890296936035156e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.378944009251427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006171076558530331 }, "reported": { "count": 6272, "observable_mse": 2.3789432816556655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006171076092869043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002879593693292197 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 2.011634933296591e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006105360575020313 }, "reported": { "count": 4092, "observable_mse": 2.0116334781050682e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006105361040681601 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028913433615768966 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.6096893634530716e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006171076092869043 }, "reported": { "count": 896, "observable_mse": 2.6096897272509523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006171076092869043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6285095230641403e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.3885084121720865e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0063759880140423775 }, "reported": { "count": 1284, "observable_mse": 3.388509503565729e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.006375988479703665 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002843561400185876 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2946816682815552, "saved_at_worst": 1.2946816682815552, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4679874330909149, "worst_tolerance_index": [ 861, 7 ], "actual_at_worst": -0.0005080029368400574, "saved_at_worst": -0.0005075111985206604, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.5174623917555436e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01194077916443348 }, "reported": { "count": 3136, "observable_mse": 3.5174627555534244e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01194077916443348 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.625227205416242e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 3.2099360396387056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011869578622281551 }, "reported": { "count": 1582, "observable_mse": 3.209937131032348e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011869577690958977 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004258530218443556 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.567524254322052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01194077916443348 }, "reported": { "count": 448, "observable_mse": 3.5675260733114555e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01194077916443348 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.812523179843284e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 3.937062501790933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012041891925036907 }, "reported": { "count": 1106, "observable_mse": 3.937062865588814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012041891925036907 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6237120250423318e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2910401821136475, "saved_at_worst": 1.2910401821136475, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.43953577764266183, "worst_tolerance_index": [ 2187, 4 ], "actual_at_worst": -0.000593923032283783, "saved_at_worst": -0.0005943886935710907, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 9.813705401029438e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022713838145136833 }, "reported": { "count": 3136, "observable_mse": 9.813706128625199e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.022713836282491684 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0005693753349948253 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 9.348728053737432e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0224068034440279 }, "reported": { "count": 1001, "observable_mse": 9.348732419312e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0224068034440279 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.32513999204833e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 9.860369755188003e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02271384373307228 }, "reported": { "count": 448, "observable_mse": 9.86036320682615e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02271384373307228 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.548361852765083e-11, "numeric_tolerance_ratio": 6.484423085950879e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00010077212209580466, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02289407141506672 }, "reported": { "count": 1687, "observable_mse": 0.00010077212937176228, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02289407141506672 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.203367743566718e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2814280986785889, "saved_at_worst": 1.2814280986785889, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.364418029785156e-07, "max_tolerance_ratio": 0.4608314751584185, "worst_tolerance_index": [ 758, 3 ], "actual_at_worst": -0.0016402006149291992, "saved_at_worst": -0.0016407370567321777, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00044241477735340595, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046979136765003204 }, "reported": { "count": 3136, "observable_mse": 0.00044241492287255824, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046979136765003204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00013935392661416896 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.00042639728053472936, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04468761011958122 }, "reported": { "count": 407, "observable_mse": 0.0004263973096385598, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04468761011958122 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.7913601978153094e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0004414815630298108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0469791404902935 }, "reported": { "count": 448, "observable_mse": 0.00044148170854896307, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046979136765003204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006537990060863823 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.00044545609853230417, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0473763607442379 }, "reported": { "count": 2281, "observable_mse": 0.0004454563313629478, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0473763607442379 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00022290136138407402 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006795178167521954 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4145125280483626e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4145105271600187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9960688147343306e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006793184205889702 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4182987544918433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4183040295611136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.262343324217096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007076622452586889 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4742850655457005e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.474279244779609e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.806399437731063e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00707575399428606 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4824970751069486e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4825025320751593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.44345482026303e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02292351797223091 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.174026930239052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.173992005642503e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00034710295461388345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02197711542248726 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.503961438080296e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.5039843573467806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.2919266484677792e-10, "numeric_tolerance_ratio": 0.0002279380971257544 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.022106481716036797 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.032768189674243e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.03276673448272e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4450289711579617e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.013866852968931198 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.329649189254269e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.3296459150733426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.26331524211703e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 6.998432908560917e-16 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006337612867355347 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7632882847683504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7633053832687438e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.709850039333105e-10, "numeric_tolerance_ratio": 0.0001705138221705848 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.003950412385165691 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.004109410336241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0041121388203464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-11, "numeric_tolerance_ratio": 2.7230268541460572e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218641808838584e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4531253780259127e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007395219057798386 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7916874387301505e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7916930775973015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.638867150992155e-11, "numeric_tolerance_ratio": 5.623168989051261e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.465091529302299e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.266129628595387e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4603585188742727e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.460366340528708e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.821654435247183e-11, "numeric_tolerance_ratio": 7.802457531363612e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.408349246252328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4083446987788193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-11, "numeric_tolerance_ratio": 4.536547937688154e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4505914552719332e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4505910914740525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6290854027335757e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.732934990490321e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7329424483468756e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.457856554538012e-11, "numeric_tolerance_ratio": 7.437530212509383e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.6724419512902386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.672434493433684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.457856554538012e-11, "numeric_tolerance_ratio": 7.437979042782642e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.5223119589791168e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.5223105947370641e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3642420526593924e-11, "numeric_tolerance_ratio": 1.3621684092581624e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5361989426310174e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5361960322479717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.90591898445941e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.4043375964974985e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.404349238029681e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011590483644883742 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 2.428454718028661e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 3.5170724004274234e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 3.5170724004274234e-06 }, "gradients": { "norms": { "prediction": 0.0001003925281111151, "weighted_latent": 2.6831793128829187e-18, "total": 0.0001003925281111151 }, "prediction_weighted_latent_dot": -2.871925118281443e-23, "prediction_weighted_latent_cosine": -0.10661591151403256, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "bda2f8457e2047e74b2f4197dcf8e137d5b22a58fb906b015a5af31e57ab3bd4" } ], "final_losses": { "prediction_mse": 2.428454718028661e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 3.5170724004274234e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 3.5170724004274234e-06 }, "training_seconds": 4.220003029331565e-05, "evaluation_seconds": 0.6724775000475347 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 2, "arm": "native", "status": "passed", "files": { "seed-2-native/result.json": "1e4139794677c6b8a539f6fb9403c61baf33ba170deeaca82a72bffbf1ec06fc", "seed-2-native/checkpoint.json": "580f3bfc531560d72eb00aabbae820bc1c8f7ce00904e4fd0199f6b3dcdc5cdd", "inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "seed-2-native/diagnostics-arrays.npz": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.172325134277344e-07, "max_tolerance_ratio": 0.38184041187270573, "worst_tolerance_index": [ 90, 15 ], "actual_at_worst": -0.0009272992610931396, "saved_at_worst": -0.0009268820285797119, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.4284559913212433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.428454718028661e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2702079400172872e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926299571990967, "saved_at_worst": 1.2926299571990967, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5302791797917392, "worst_tolerance_index": [ 545, 1 ], "actual_at_worst": -0.0012408345937728882, "saved_at_worst": -0.0012402385473251343, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.00010886224481509998, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.048658352345228195 }, "reported": { "count": 784, "observable_mse": 0.00010886227391893044, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0486583486199379 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006350827096410438 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 0.00011503381392685696, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05235643684864044 }, "reported": { "count": 580, "observable_mse": 0.00011503385758260265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05235643684864044 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3159267976522494e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.00013099024363327771, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.048658352345228195 }, "reported": { "count": 112, "observable_mse": 0.00013099025818519294, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.048658352345228195 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4363763914006169e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 4.30161198892165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "reported": { "count": 92, "observable_mse": 4.301610169932246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.811198334370962e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4489330826327205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 98, "observable_mse": 2.448930717946496e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749081823974848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333975094181 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0924091586493887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925644695759 }, "reported": { "count": 70, "observable_mse": 2.0924004275002517e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749084152281284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333754161296 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1905292669543996e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 14, "observable_mse": 2.1905329049332067e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474907483905554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.792795956134796e-08, "numeric_tolerance_ratio": 0.01330469955564529 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.489958519116044e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474924247711897 }, "reported": { "count": 14, "observable_mse": 4.489981074584648e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00347490725107491 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6996636986732483e-08, "numeric_tolerance_ratio": 0.012613546549922739 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4650891646160744e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355574123561382 }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017459385640157722 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0494757336564362e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 70, "observable_mse": 2.049479189736303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355571795254946 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00034918772499636073 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.5227938749594614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335555549710989 }, "reported": { "count": 14, "observable_mse": 2.5227842343156226e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355604391545057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.0036664702162613125 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.485453246161342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 14, "observable_mse": 4.485439421841875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355567138642073 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006983754743791358 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4724424292799085e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488371381536126 }, "reported": { "count": 98, "observable_mse": 2.4724389732000418e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883692860603333 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0954757928848267e-09, "numeric_tolerance_ratio": 0.0015535427214692392 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9836363208014518e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488370683044195 }, "reported": { "count": 70, "observable_mse": 1.983637412195094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012083110264445502 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.7921978471567854e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883730113506317 }, "reported": { "count": 14, "observable_mse": 2.792183840938378e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488372080028057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006904632887422107 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.596718281391077e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883723128587008 }, "reported": { "count": 14, "observable_mse": 4.596703365677968e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0024166220528891004 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4542685423512012e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660863498225808 }, "reported": { "count": 98, "observable_mse": 2.4542630853829905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660864196717739 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0005113087436513964 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9375931515241973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036534646060317755 }, "reported": { "count": 72, "observable_mse": 1.9375880583538674e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00365346553735435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006821144214759868 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0936709663365036e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608625669032335 }, "reported": { "count": 14, "observable_mse": 3.093665145570412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608658265322447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002386107185697774 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8083511501317844e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704948117956519 }, "reported": { "count": 12, "observable_mse": 4.808346784557216e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704944159835577 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.958120942115784e-09, "numeric_tolerance_ratio": 0.002888097095437761 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.454017158015631e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00366382603533566 }, "reported": { "count": 98, "observable_mse": 2.4540126105421223e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638197489082813 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004600783305236997 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9373417671886273e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656336572021246 }, "reported": { "count": 72, "observable_mse": 1.937336492119357e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656331915408373 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0034098562497762368 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.09398710669484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638290621340275 }, "reported": { "count": 14, "observable_mse": 3.0939725547796115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003663820680230856 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.0061343773221983556 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8074376536533237e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003708441974595189 }, "reported": { "count": 12, "observable_mse": 4.8074492951855063e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00370843056589365 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1408701539039612e-08, "numeric_tolerance_ratio": 0.008322398019379602 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4505963665433228e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 98, "observable_mse": 2.4505910914740525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0662389943026938e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 70, "observable_mse": 2.066234446829185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.32611782848835e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416656516492367 }, "reported": { "count": 14, "observable_mse": 2.3261030946741812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416670486330986 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.010412299112111879 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.4968615839025006e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416659077629447 }, "reported": { "count": 14, "observable_mse": 4.4968630390940234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416667925193906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006594457363167671 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0007092258892953396, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "reported": { "count": 98, "observable_mse": 0.0007092263549566269, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00043482252767233174 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.000681952282320708, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "reported": { "count": 84, "observable_mse": 0.0006819527479819953, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0004359327346731717 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0008728672983124852, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "reported": { "count": 14, "observable_mse": 0.0008728685788810253, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1373080313205719 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.280568540096283e-09, "numeric_tolerance_ratio": 0.001177765123165015 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218651813280303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703649388626218 }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703637048602104 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.009004926261465663 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.265882656298345e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003696256782859564 }, "reported": { "count": 72, "observable_mse": 1.2658831110456958e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036962456069886684 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.008159806136714668 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4861635463603307e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703654045239091 }, "reported": { "count": 14, "observable_mse": 1.4861591807857621e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037036393769085407 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4668330550193787e-08, "numeric_tolerance_ratio": 0.010703967133658529 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.2827452994533814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037476930301636457 }, "reported": { "count": 12, "observable_mse": 2.2827287466498092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00374767673201859 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-08, "numeric_tolerance_ratio": 0.01185519951732077 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29456627368927, "saved_at_worst": 1.29456627368927, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.48837328246516154, "worst_tolerance_index": [ 1481, 14 ], "actual_at_worst": 6.841123104095459e-05, "saved_at_worst": 6.890296936035156e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.0004635656950995326, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07253267616033554 }, "reported": { "count": 6272, "observable_mse": 0.00046356572420336306, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07253267616033554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.7814447984412602e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.0005398132489062846, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07738631963729858 }, "reported": { "count": 4092, "observable_mse": 0.0005398132489062846, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07738631963729858 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.00046595203457400203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07253267616033554 }, "reported": { "count": 896, "observable_mse": 0.0004659521218854934, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.07253267616033554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.342431759039187e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.00021890609059482813, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.054240379482507706 }, "reported": { "count": 1284, "observable_mse": 0.00021890613425057381, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.054240379482507706 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.272056628329344e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2131738662719727, "saved_at_worst": 1.2131738662719727, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.39050032273574337, "worst_tolerance_index": [ 1143, 0 ], "actual_at_worst": -0.00020797178149223328, "saved_at_worst": -0.00020757317543029785, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0008681408362463117, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10035797953605652 }, "reported": { "count": 3136, "observable_mse": 0.0008681408944539726, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10035797208547592 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0006751284439290988 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.000930042820982635, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10425479710102081 }, "reported": { "count": 1582, "observable_mse": 0.0009300428791902959, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10425479710102081 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.3254741593273114e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0008559529087506235, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10035797208547592 }, "reported": { "count": 448, "observable_mse": 0.0008559530833736062, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.10035797208547592 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016085458494459172 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.0007845343207009137, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09450514614582062 }, "reported": { "count": 1106, "observable_mse": 0.0007845344371162355, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09450514614582062 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010794654373421804 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2567524909973145, "saved_at_worst": 1.2567524909973145, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.3373281591691426, "worst_tolerance_index": [ 1728, 1 ], "actual_at_worst": 0.00015969574451446533, "saved_at_worst": 0.00016003847122192383, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0070852432399988174, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.20626290142536163 }, "reported": { "count": 3136, "observable_mse": 0.00708524277433753, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.20626290142536163 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002725517532634505 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.008809944614768028, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.22894664108753204 }, "reported": { "count": 1001, "observable_mse": 0.008809943683445454, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.22894664108753204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004951224683544009 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0070871817879378796, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.20626290142536163 }, "reported": { "count": 448, "observable_mse": 0.007087182253599167, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.20626290142536163 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002725208172983901 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.006061358377337456, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1915375143289566 }, "reported": { "count": 1687, "observable_mse": 0.00606135930866003, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1915374994277954 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0007393741232353771 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2043378353118896, "saved_at_worst": 1.2043378353118896, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.736947059631348e-07, "max_tolerance_ratio": 0.3696263958661877, "worst_tolerance_index": [ 1766, 3 ], "actual_at_worst": -0.0006836354732513428, "saved_at_worst": -0.0006832405924797058, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0199440885335207, "exact_state_accuracy": 0.9205994897959183, "per_field_accuracy": { "x": 0.9652423469387755, "y": 0.9719387755102041, "h": 0.9792729591836735 }, "normalized_latent_rmse": 0.29997682571411133 }, "reported": { "count": 3136, "observable_mse": 0.0199440885335207, "exact_state_accuracy": 0.9205994606018066, "per_field_accuracy": { "x": 0.965242326259613, "y": 0.9719387292861938, "h": 0.9792729616165161 }, "normalized_latent_rmse": 0.29997679591178894 }, "differences": [], "max_errors": { "accuracy_abs": 4.622401028608891e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0009614372037117338 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.025644240900874138, "exact_state_accuracy": 0.8771498771498771, "per_field_accuracy": { "x": 0.9656019656019657, "y": 0.9533169533169533, "h": 0.9533169533169533 }, "normalized_latent_rmse": 0.3439321219921112 }, "reported": { "count": 407, "observable_mse": 0.025644240900874138, "exact_state_accuracy": 0.8771498799324036, "per_field_accuracy": { "x": 0.965601921081543, "y": 0.9533169269561768, "h": 0.9533169269561768 }, "normalized_latent_rmse": 0.3439320921897888 }, "differences": [], "max_errors": { "accuracy_abs": 4.452042268177081e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0008420350413353993 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.020050769671797752, "exact_state_accuracy": 0.9196428571428571, "per_field_accuracy": { "x": 0.9665178571428571, "y": 0.9709821428571429, "h": 0.9799107142857143 }, "normalized_latent_rmse": 0.29997679591178894 }, "reported": { "count": 448, "observable_mse": 0.0200507715344429, "exact_state_accuracy": 0.9196429252624512, "per_field_accuracy": { "x": 0.9665179252624512, "y": 0.9709821939468384, "h": 0.9799107313156128 }, "normalized_latent_rmse": 0.29997679591178894 }, "differences": [], "max_errors": { "accuracy_abs": 6.811959407659884e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006198327211319927 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.01890605501830578, "exact_state_accuracy": 0.9285401139850943, "per_field_accuracy": { "x": 0.9649276633055678, "y": 0.9754493643138974, "h": 0.9837790442788251 }, "normalized_latent_rmse": 0.29143768548965454 }, "reported": { "count": 2281, "observable_mse": 0.01890605501830578, "exact_state_accuracy": 0.9285401105880737, "per_field_accuracy": { "x": 0.9649276733398438, "y": 0.9754493832588196, "h": 0.9837790727615356 }, "normalized_latent_rmse": 0.29143771529197693 }, "differences": [], "max_errors": { "accuracy_abs": 2.8482710523469734e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0009886726469787747 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006795178167521954 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4145125280483626e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4145105271600187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9960688147343306e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006793184205889702 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4182987544918433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4183040295611136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.262343324217096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007076622452586889 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4742850655457005e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.474279244779609e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.806399437731063e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00707575399428606 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4824970751069486e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4825025320751593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.44345482026303e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02292351797223091 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.174026930239052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.173992005642503e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00034710295461388345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02197711542248726 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.503961438080296e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.5039843573467806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.2919266484677792e-10, "numeric_tolerance_ratio": 0.0002279380971257544 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.022106481716036797 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.032768189674243e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.03276673448272e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4450289711579617e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.013866852968931198 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.329649189254269e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.3296459150733426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.26331524211703e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.23434017598628998 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.011553759686648846, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.01155376061797142, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.000432092845013295 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.2341216802597046 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.013623347505927086, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.013623349368572235, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0007884763164485735 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.003950412385165691 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.004109410336241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0041121388203464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-11, "numeric_tolerance_ratio": 2.7230268541460572e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218641808838584e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4531253780259127e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007395219057798386 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7916874387301505e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7916930775973015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.638867150992155e-11, "numeric_tolerance_ratio": 5.623168989051261e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.465091529302299e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.266129628595387e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4603585188742727e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.460366340528708e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.821654435247183e-11, "numeric_tolerance_ratio": 7.802457531363612e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.408349246252328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4083446987788193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-11, "numeric_tolerance_ratio": 4.536547937688154e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0020825800020247698, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0020825802348554134, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00019269943929874165 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0007546067936345935, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0007546074921265244, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0006494815654341421 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.6724419512902386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.672434493433684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.457856554538012e-11, "numeric_tolerance_ratio": 7.437979042782642e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.5223119589791168e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.5223105947370641e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3642420526593924e-11, "numeric_tolerance_ratio": 1.3621684092581624e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5361989426310174e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5361960322479717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.90591898445941e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.4043375964974985e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.404349238029681e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011590483644883742 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 636, "error_count": 50, "per_field_correct_counts": { "x": 682, "y": 640, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 441, "error_count": 49, "per_field_correct_counts": { "x": 441, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 91, "error_count": 7, "per_field_correct_counts": { "x": 98, "y": 91, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 140, "error_count": 56, "per_field_correct_counts": { "x": 140, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 74, "error_count": 24, "per_field_correct_counts": { "x": 98, "y": 84, "h": 84 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 35, "error_count": 63, "per_field_correct_counts": { "x": 98, "y": 77, "h": 47 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 2887, "accuracy": 0.9205994897959183 }, "numpy": { "cases": 3136, "correct": 2887, "accuracy": 0.9205994897959183 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0006819527479819953, "latent_mse": 0.01885349676012993, "scaled_prediction": 9.876557305688038e-05, "scaled_weighted_latent": 0.0002730506530497223, "total": 0.0003718162188306451 }, "gradients": { "norms": { "prediction": 0.0009655424510128796, "weighted_latent": 0.001503756851889193, "total": 0.002251012483611703 }, "prediction_weighted_latent_dot": 9.367491315970256e-07, "prediction_weighted_latent_cosine": 0.6451702116278689, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "580f3bfc531560d72eb00aabbae820bc1c8f7ce00904e4fd0199f6b3dcdc5cdd" } ], "final_losses": { "prediction_mse": 0.0006819527479819953, "latent_mse": 0.01885349676012993, "scaled_prediction": 9.876557305688038e-05, "scaled_weighted_latent": 0.0002730506530497223, "total": 0.0003718162188306451 }, "training_seconds": 4.299997817724943e-05, "evaluation_seconds": 0.5754795999964699 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 2, "arm": "prediction_latent", "status": "passed", "files": { "seed-2-prediction_latent/result.json": "3c6dd3dc32279410c2d961c6fca25eccac4ec596e4d54b8e89de094c21fd5072", "seed-2-prediction_latent/checkpoint.json": "96fb11342d85934fa4a7978a0ab5c3a67fc936118bd303aa84569e9dc2413884", "inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "seed-2-prediction_latent/diagnostics-arrays.npz": "76c4e3d623aa6a3635ab5ebfaa25058b43e9e885408a610ea856200f7b23f635" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5", "parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34", "parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.172325134277344e-07, "max_tolerance_ratio": 0.38184041187270573, "worst_tolerance_index": [ 90, 15 ], "actual_at_worst": -0.0009272992610931396, "saved_at_worst": -0.0009268820285797119, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.4284559913212433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.428454718028661e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2702079400172872e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 4.753383473143913e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2926299571990967, "saved_at_worst": 1.2926299571990967, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5302791797917392, "worst_tolerance_index": [ 545, 1 ], "actual_at_worst": -0.0012408345937728882, "saved_at_worst": -0.0012402385473251343, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.326729372725822e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034997230395674706 }, "reported": { "count": 784, "observable_mse": 2.3267271899385378e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034997230395674706 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1777203231670335e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.9321074432809837e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00349230645224452 }, "reported": { "count": 580, "observable_mse": 1.9321063518873416e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034923062194138765 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017256549018940373 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.748085171333514e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034997237380594015 }, "reported": { "count": 112, "observable_mse": 2.7480740754981525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034997237380594015 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1095835361629725e-10, "numeric_tolerance_ratio": 0.000110654267492458 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 4.30161198892165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "reported": { "count": 92, "observable_mse": 4.301610169932246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035461238585412502 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.811198334370962e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4489330826327205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 98, "observable_mse": 2.448930717946496e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749081823974848 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333975094181 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0924091586493887e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925644695759 }, "reported": { "count": 70, "observable_mse": 2.0924004275002517e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034749084152281284 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7229467630386353e-08, "numeric_tolerance_ratio": 0.012786333754161296 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1905292669543996e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474925411865115 }, "reported": { "count": 14, "observable_mse": 2.1905329049332067e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474907483905554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.792795956134796e-08, "numeric_tolerance_ratio": 0.01330469955564529 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.489958519116044e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003474924247711897 }, "reported": { "count": 14, "observable_mse": 4.489981074584648e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00347490725107491 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6996636986732483e-08, "numeric_tolerance_ratio": 0.012613546549922739 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4650891646160744e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355574123561382 }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017459385640157722 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0494757336564362e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 70, "observable_mse": 2.049479189736303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355571795254946 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00034918772499636073 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.5227938749594614e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335555549710989 }, "reported": { "count": 14, "observable_mse": 2.5227842343156226e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355604391545057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.0036664702162613125 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.485453246161342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003335557645186782 }, "reported": { "count": 14, "observable_mse": 4.485439421841875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033355567138642073 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006983754743791358 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4724424292799085e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488371381536126 }, "reported": { "count": 98, "observable_mse": 2.4724389732000418e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883692860603333 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0954757928848267e-09, "numeric_tolerance_ratio": 0.0015535427214692392 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9836363208014518e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488370683044195 }, "reported": { "count": 70, "observable_mse": 1.983637412195094e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012083110264445502 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.7921978471567854e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883730113506317 }, "reported": { "count": 14, "observable_mse": 2.792183840938378e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003488372080028057 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006904632887422107 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.596718281391077e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883723128587008 }, "reported": { "count": 14, "observable_mse": 4.596703365677968e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034883690532296896 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0024166220528891004 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4542685423512012e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660863498225808 }, "reported": { "count": 98, "observable_mse": 2.4542630853829905e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003660864196717739 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-10, "numeric_tolerance_ratio": 0.0005113087436513964 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9375931515241973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036534646060317755 }, "reported": { "count": 72, "observable_mse": 1.9375880583538674e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00365346553735435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0006821144214759868 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.0936709663365036e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608625669032335 }, "reported": { "count": 14, "observable_mse": 3.093665145570412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036608658265322447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.002386107185697774 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8083511501317844e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704948117956519 }, "reported": { "count": 12, "observable_mse": 4.808346784557216e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003704944159835577 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.958120942115784e-09, "numeric_tolerance_ratio": 0.002888097095437761 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.454017158015631e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00366382603533566 }, "reported": { "count": 98, "observable_mse": 2.4540126105421223e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638197489082813 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004600783305236997 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.9373417671886273e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656336572021246 }, "reported": { "count": 72, "observable_mse": 1.937336492119357e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003656331915408373 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.6566128730773926e-09, "numeric_tolerance_ratio": 0.0034098562497762368 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 3.09398710669484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036638290621340275 }, "reported": { "count": 14, "observable_mse": 3.0939725547796115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003663820680230856 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.0061343773221983556 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.8074376536533237e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003708441974595189 }, "reported": { "count": 12, "observable_mse": 4.8074492951855063e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00370843056589365 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1408701539039612e-08, "numeric_tolerance_ratio": 0.008322398019379602 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4505963665433228e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 98, "observable_mse": 2.4505910914740525e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0662389943026938e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034166586119681597 }, "reported": { "count": 70, "observable_mse": 2.066234446829185e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416669089347124 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.00780922514720379 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.32611782848835e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416656516492367 }, "reported": { "count": 14, "observable_mse": 2.3261030946741812e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416670486330986 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.010412299112111879 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 4.4968615839025006e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416659077629447 }, "reported": { "count": 14, "observable_mse": 4.4968630390940234e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003416667925193906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006594457363167671 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4466247850796208e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032246853224933147 }, "reported": { "count": 98, "observable_mse": 2.4466211471008137e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032246781047433615 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.2177499532699585e-09, "numeric_tolerance_ratio": 0.005457788761362088 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.1078587451484054e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032246853224933147 }, "reported": { "count": 84, "observable_mse": 2.1078567442600615e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032246774062514305 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.916241884231567e-09, "numeric_tolerance_ratio": 0.005985962183462777 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 4.479219933273271e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032246848568320274 }, "reported": { "count": 14, "observable_mse": 4.47920938313473e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003224681131541729 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0028169225869475625 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218651813280303e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703649388626218 }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703637048602104 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.009004926261465663 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.265882656298345e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003696256782859564 }, "reported": { "count": 72, "observable_mse": 1.2658831110456958e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0036962456069886684 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.008159806136714668 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4861635463603307e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003703654045239091 }, "reported": { "count": 14, "observable_mse": 1.4861591807857621e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037036393769085407 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4668330550193787e-08, "numeric_tolerance_ratio": 0.010703967133658529 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 2.2827452994533814e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0037476930301636457 }, "reported": { "count": 12, "observable_mse": 2.2827287466498092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00374767673201859 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-08, "numeric_tolerance_ratio": 0.01185519951732077 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29456627368927, "saved_at_worst": 1.29456627368927, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5081765317664586, "worst_tolerance_index": [ 4802, 10 ], "actual_at_worst": -0.00041012465953826904, "saved_at_worst": -0.00040959566831588745, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.381150443397928e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006720003671944141 }, "reported": { "count": 6272, "observable_mse": 2.381150443397928e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006720003206282854 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00027850550120274995 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 2.00920276256511e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006727665662765503 }, "reported": { "count": 4092, "observable_mse": 2.0092016711714678e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006727665662765503 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0892052092009387e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.616950405354146e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006720003671944141 }, "reported": { "count": 896, "observable_mse": 2.6169513148488477e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006720003671944141 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.071208107744453e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.401971480343491e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006695527117699385 }, "reported": { "count": 1284, "observable_mse": 3.401972571737133e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.006695527117699385 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0876933392210228e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2936732769012451, "saved_at_worst": 1.2936732769012451, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4389024947856207, "worst_tolerance_index": [ 893, 11 ], "actual_at_worst": 0.00026971474289894104, "saved_at_worst": 0.00027016550302505493, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.829687193501741e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013225768692791462 }, "reported": { "count": 3136, "observable_mse": 3.8296886486932635e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013225768692791462 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4496398535448712e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 3.53561008523684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013272375799715519 }, "reported": { "count": 1582, "observable_mse": 3.535611540428363e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013272375799715519 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4500646574992635e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.879581345245242e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013225769624114037 }, "reported": { "count": 448, "observable_mse": 3.879580617649481e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.013225769624114037 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.247839038330476e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 4.2301195207983255e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01315881498157978 }, "reported": { "count": 1106, "observable_mse": 4.2301209759898484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01315881498157978 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4490618160530932e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2936776876449585, "saved_at_worst": 1.2936776876449585, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.556510925292969e-07, "max_tolerance_ratio": 0.4808236477509496, "worst_tolerance_index": [ 1653, 3 ], "actual_at_worst": -0.00038145482540130615, "saved_at_worst": -0.00038195401430130005, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0001303325843764469, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026330674067139626 }, "reported": { "count": 3136, "observable_mse": 0.00013033264258410782, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026330672204494476 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.000512692178869327 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00013152690371498466, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02676781266927719 }, "reported": { "count": 1001, "observable_mse": 0.00013152696192264557, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02676781266927719 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.745201205329609e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0001308084320044145, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026330670341849327 }, "reported": { "count": 448, "observable_mse": 0.00013080847566016018, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.026330670341849327 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3092064952156534e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00012949758092872798, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02606782503426075 }, "reported": { "count": 1687, "observable_mse": 0.00012949762458447367, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02606782503426075 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.309764146560169e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2798446416854858, "saved_at_worst": 1.2798446416854858, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4056835508193834, "worst_tolerance_index": [ 1762, 3 ], "actual_at_worst": -0.0010197460651397705, "saved_at_worst": -0.001019299030303955, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0005900115356780589, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05305289849638939 }, "reported": { "count": 3136, "observable_mse": 0.0005900117103010416, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05305289849638939 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016489404121295182 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0006105774082243443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.052701715379953384 }, "reported": { "count": 407, "observable_mse": 0.000610577582847327, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.052701715379953384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016457443657232326 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0005885306745767593, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05305289477109909 }, "reported": { "count": 448, "observable_mse": 0.0005885308491997421, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05305289477109909 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.0001649171025020906 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0005866328137926757, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05311531946063042 }, "reported": { "count": 2281, "observable_mse": 0.0005866330466233194, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.05311531573534012 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0005902355482277996 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006795178167521954 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4145125280483626e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4145105271600187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9960688147343306e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006793184205889702 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4182987544918433e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4183040295611136e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.262343324217096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007076622452586889 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4742850655457005e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.474279244779609e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.806399437731063e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00707575399428606 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4824970751069486e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4825025320751593e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-11, "numeric_tolerance_ratio": 5.44345482026303e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02292351797223091 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.174026930239052e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.173992005642503e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00034710295461388345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.02197711542248726 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 5.503961438080296e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 5.5039843573467806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.2919266484677792e-10, "numeric_tolerance_ratio": 0.0002279380971257544 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.022106481716036797 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.032768189674243e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.03276673448272e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4450289711579617e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.013866852968931198 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.329649189254269e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.3296459150733426e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.26331524211703e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006114483345299959 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7247950129094534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7247851903666742e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.822542779147625e-11, "numeric_tolerance_ratio": 9.795851188901073e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012204460799694061 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.575491766445339e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.575491766445339e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7605883587966673e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7605872674030252e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0883890491763767e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.003950412385165691 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.004109410336241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.0041121388203464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7284841053187847e-11, "numeric_tolerance_ratio": 2.7230268541460572e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4218641808838584e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4218627256923355e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4531253780259127e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.007395219057798386 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7916874387301505e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7916930775973015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.638867150992155e-11, "numeric_tolerance_ratio": 5.623168989051261e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.465091529302299e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4650882551213726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.2741809263825417e-11, "numeric_tolerance_ratio": 3.266129628595387e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.4603585188742727e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.460366340528708e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.821654435247183e-11, "numeric_tolerance_ratio": 7.802457531363612e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.408349246252328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.4083446987788193e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.547473508864641e-11, "numeric_tolerance_ratio": 4.536547937688154e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.468158709234558e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.468152524670586e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.184563972055912e-11, "numeric_tolerance_ratio": 6.169337107099482e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7641772248898633e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7641910492093302e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.00013786211743843673 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.6724419512902386e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.672434493433684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.457856554538012e-11, "numeric_tolerance_ratio": 7.437979042782642e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.5223119589791168e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.5223105947370641e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3642420526593924e-11, "numeric_tolerance_ratio": 1.3621684092581624e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5361989426310174e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.5361960322479717e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.90591898445941e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 4.4043375964974985e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.404349238029681e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011590483644883742 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0006819527479819953, "latent_mse": 0.01885349676012993, "scaled_prediction": 9.876557305688038e-05, "scaled_weighted_latent": 0.0002730506530497223, "total": 0.0003718162188306451 }, "gradients": { "norms": { "prediction": 0.0009655424510128796, "weighted_latent": 0.001503756851889193, "total": 0.002251012483611703 }, "prediction_weighted_latent_dot": 9.367491315970256e-07, "prediction_weighted_latent_cosine": 0.6451702116278689, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0007664921577088535, "latent_mse": 0.017893053591251373, "scaled_prediction": 0.00011100921256002039, "scaled_weighted_latent": 0.000259140768321231, "total": 0.00037014996632933617 }, "gradients": { "norms": { "prediction": 0.0016901097260415554, "weighted_latent": 0.0014657382853329182, "total": 0.0026715160347521305 }, "prediction_weighted_latent_dot": 1.0660676252882695e-06, "prediction_weighted_latent_cosine": 0.4303416908538793, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.02936518006026745 }, { "step": 100, "losses": { "prediction_mse": 2.6542955311015248e-05, "latent_mse": 0.000432105443906039, "scaled_prediction": 3.844152161036618e-06, "scaled_weighted_latent": 6.258078883547569e-06, "total": 1.0102230589836836e-05 }, "gradients": { "norms": { "prediction": 0.00014162296429276466, "weighted_latent": 0.0002406078710919246, "total": 0.0003380496636964381 }, "prediction_weighted_latent_dot": 1.8164215020988195e-08, "prediction_weighted_latent_cosine": 0.5330563527859227, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.001268652849830687 }, { "step": 200, "losses": { "prediction_mse": 2.0889330698992126e-05, "latent_mse": 1.2319784218561836e-05, "scaled_prediction": 3.0253513614297844e-06, "scaled_weighted_latent": 1.7842447164184705e-07, "total": 3.203775804649922e-06 }, "gradients": { "norms": { "prediction": 3.2663530873833224e-05, "weighted_latent": 3.844348975690082e-05, "total": 6.885149105073651e-06 }, "prediction_weighted_latent_dot": -1.2487013645312572e-09, "prediction_weighted_latent_cosine": -0.9944264160208769, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.3797200962435454e-05 }, { "step": 300, "losses": { "prediction_mse": 2.107463478751015e-05, "latent_mse": 1.0362839930166956e-05, "scaled_prediction": 3.0521885037160246e-06, "scaled_weighted_latent": 1.5008251352810476e-07, "total": 3.2022710456658388e-06 }, "gradients": { "norms": { "prediction": 3.676263804663904e-05, "weighted_latent": 3.492542600724846e-05, "total": 1.2157050150563009e-05 }, "prediction_weighted_latent_dot": -1.211741595952276e-09, "prediction_weighted_latent_cosine": -0.9437601508857996, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00044254420208744705 }, { "step": 400, "losses": { "prediction_mse": 2.1073601601528935e-05, "latent_mse": 1.0396162906545214e-05, "scaled_prediction": 3.052038891837583e-06, "scaled_weighted_latent": 1.5056512836508773e-07, "total": 3.2026039207266876e-06 }, "gradients": { "norms": { "prediction": 3.8248101191129535e-05, "weighted_latent": 3.5035762266488746e-05, "total": 1.6219282770180143e-05 }, "prediction_weighted_latent_dot": -1.2136783800187345e-09, "prediction_weighted_latent_cosine": -0.9056954066841081, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005452635232359171 }, { "step": 500, "losses": { "prediction_mse": 2.1075411495985463e-05, "latent_mse": 1.0414029020466842e-05, "scaled_prediction": 3.052301053685369e-06, "scaled_weighted_latent": 1.508238653968874e-07, "total": 3.203124833817128e-06 }, "gradients": { "norms": { "prediction": 3.8674745155731216e-05, "weighted_latent": 3.5055403714068234e-05, "total": 1.786229040590115e-05 }, "prediction_weighted_latent_dot": -1.2027779883183598e-09, "prediction_weighted_latent_cosine": -0.8871622178713677, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005519929109141231 }, { "step": 600, "losses": { "prediction_mse": 2.1348816517274827e-05, "latent_mse": 1.0642489542078692e-05, "scaled_prediction": 3.091897724516457e-06, "scaled_weighted_latent": 1.5413260712193733e-07, "total": 3.246030246373266e-06 }, "gradients": { "norms": { "prediction": 8.918772800825536e-05, "weighted_latent": 3.561050471034832e-05, "total": 8.645343041280285e-05 }, "prediction_weighted_latent_dot": -8.741819490509783e-10, "prediction_weighted_latent_cosine": -0.27524447161999827, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0025028504896909 }, { "step": 700, "losses": { "prediction_mse": 2.1100107915117405e-05, "latent_mse": 1.03705060610082e-05, "scaled_prediction": 3.0558778689737665e-06, "scaled_weighted_latent": 1.5019354293599463e-07, "total": 3.206071369277197e-06 }, "gradients": { "norms": { "prediction": 4.425676161190495e-05, "weighted_latent": 3.4928660170407966e-05, "total": 2.9300932510523126e-05 }, "prediction_weighted_latent_dot": -1.1600639338027463e-09, "prediction_weighted_latent_cosine": -0.75044758746988, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0008021992980502546 }, { "step": 800, "losses": { "prediction_mse": 2.106979809468612e-05, "latent_mse": 1.0369310075475369e-05, "scaled_prediction": 3.051487965421984e-06, "scaled_weighted_latent": 1.501762199040968e-07, "total": 3.2016641853260808e-06 }, "gradients": { "norms": { "prediction": 3.6618766898754984e-05, "weighted_latent": 3.494641714496538e-05, "total": 1.1658473340503406e-05 }, "prediction_weighted_latent_dot": -1.213133149491341e-09, "prediction_weighted_latent_cosine": -0.9479863801021514, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0002748027036432177 }, { "step": 900, "losses": { "prediction_mse": 2.1071549781481735e-05, "latent_mse": 1.0367329196014907e-05, "scaled_prediction": 3.0517417144437786e-06, "scaled_weighted_latent": 1.501475281884268e-07, "total": 3.2018892852647696e-06 }, "gradients": { "norms": { "prediction": 3.6724366509588435e-05, "weighted_latent": 3.4931050322484225e-05, "total": 1.1596196600294206e-05 }, "prediction_weighted_latent_dot": -1.217192679980883e-09, "prediction_weighted_latent_cosine": -0.9488408513342776, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00042341844527982175 }, { "step": 1000, "losses": { "prediction_mse": 2.1078567442600615e-05, "latent_mse": 1.0398544873169158e-05, "scaled_prediction": 3.05275807477301e-06, "scaled_weighted_latent": 1.5059961810948153e-07, "total": 3.203357664460782e-06 }, "gradients": { "norms": { "prediction": 4.1895014874171466e-05, "weighted_latent": 3.5039363865507767e-05, "total": 2.398364267719444e-05 }, "prediction_weighted_latent_dot": -1.2038667840386097e-09, "prediction_weighted_latent_cosine": -0.8200868913974374, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00048400292871519923 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "363d6312324e6bb76c3505974198defafeb076e7a6dfcca3f6e995b0d6f68314" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "1a29db21b70eb2808fb52550476aa65f77fa3d52d5de26d2249720f212fd80a5" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "6e15fc9c2fc9d48920f0431193a8cc1071fcf7115c1ca38d71c03dd8b2ae300a" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "ad49ebd86692985f6c1dadc11727748c9564df2512f101feeb2fc3eea7d62bf6" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "86c8a5a1f8fd4080cd433c2018d0993cbebc742afe86cdfa587daa67f2437e14" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "fb0f041f35ad073664da9a86fae79a5c8b697268d0036fde78cd5d8a3d85102b" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "841fb220944a4bf864e696696022cc9c9eba9455fcfb870b8e712db105fc6e0c" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "4d1a6f7e75cc98493e3ecece3c766a78797c457bb6a1b16f8b0693871fed9949" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "f05cda5d84c6b8475222308b8c1ae14fc30e1b919a04f68afad3e408caefbd77" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "2df653451c9b3123ba08535fba3920cab58b1bf22e650e5b747eb3eae7150e64" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "5c34ee19a38584fb4f53325f797a694edd412edb11a1396bebd0abcaa30380d1" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "96fb11342d85934fa4a7978a0ab5c3a67fc936118bd303aa84569e9dc2413884" } ], "final_losses": { "prediction_mse": 2.1078567442600615e-05, "latent_mse": 1.0398544873169158e-05, "scaled_prediction": 3.05275807477301e-06, "scaled_weighted_latent": 1.5059961810948153e-07, "total": 3.203357664460782e-06 }, "training_seconds": 3.3037349000223912, "evaluation_seconds": 0.507699900015723 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 3, "arm": "refit", "status": "passed", "files": { "seed-3-refit/result.json": "9532d71b6cfd1249296a2382024f3026133c0435bd7b943d96faf9d9d97cd910", "seed-3-refit/checkpoint.json": "e10d1479ab2365c43ffe7e27c9b1efa8384babeabb8f6328e4de58db8d11fd16", "inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "seed-3-refit/diagnostics-arrays.npz": "cb4c22da61b396601351d7f1dae4ffed51547a2fcfacb19a754fd123b21b0682" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 5.811452865600586e-07, "max_tolerance_ratio": 0.3990840958602454, "worst_tolerance_index": [ 60, 2 ], "actual_at_worst": 0.004562556743621826, "saved_at_worst": 0.004561975598335266, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 0.00010696637036744505, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 0.00010696651588659734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014397905845926648 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001894764427561313, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 0.00018947666103485972, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-10, "numeric_tolerance_ratio": 0.00021421976386698359 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2893109321594238, "saved_at_worst": 1.2893109321594238, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.3990840958602454, "worst_tolerance_index": [ 649, 2 ], "actual_at_worst": 0.004562556743621826, "saved_at_worst": 0.004561975598335266, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.00010002128692576662, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "reported": { "count": 784, "observable_mse": 0.00010002134513342753, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.763122564242308e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 8.43349116621539e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005999956279993057 }, "reported": { "count": 580, "observable_mse": 8.433497714577243e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0059999567456543446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002910390913614281 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.00011633474059635773, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "reported": { "count": 112, "observable_mse": 0.00011633482063189149, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.911514908817387e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 0.00017905373533722013, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "reported": { "count": 92, "observable_mse": 0.00017905370623338968, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8591882208963364e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.000102343074104283, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "reported": { "count": 98, "observable_mse": 0.00010234319051960483, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001757528884955003 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.67251874296926e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897133611142635 }, "reported": { "count": 70, "observable_mse": 8.67253402248025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0020504503657808367 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010630184988258407, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897137802094221 }, "reported": { "count": 14, "observable_mse": 0.00010630159522406757, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002343372258461114 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017647375352680683, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897135008126497 }, "reported": { "count": 14, "observable_mse": 0.00017647400090936571, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897139199078083 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002636292941319088 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252822539769113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.737066673347726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 70, "observable_mse": 8.737093594390899e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010268489859299734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 14, "observable_mse": 0.0001026846730383113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.255546860396862e-10, "numeric_tolerance_ratio": 0.00022326212619665206 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017815936007536948, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "reported": { "count": 14, "observable_mse": 0.00017815965111367404, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002859439373556026 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001030568455462344, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580707758665 }, "reported": { "count": 98, "observable_mse": 0.00010305678733857349, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00579952893372632 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.592774247517809e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580242097378 }, "reported": { "count": 70, "observable_mse": 8.592770609539002e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585904866456985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.006379481642109011 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001099406072171405, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585797764360905 }, "reported": { "count": 14, "observable_mse": 0.00010994031617883593, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585867613554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.004349647583201486 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00018181865743827075, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058583967387676 }, "reported": { "count": 14, "observable_mse": 0.00018181874474976212, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003769693806922108 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200838732998818, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097275298088789 }, "reported": { "count": 98, "observable_mse": 0.00010200848191743717, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097279489040375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002603515450311345 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543613512301818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060906685888767 }, "reported": { "count": 72, "observable_mse": 8.543617150280625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060910876840353 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026094108969952367 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001222111313836649, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00609727343544364 }, "reported": { "count": 14, "observable_mse": 0.00012221158249303699, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060972776263952255 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026035157515688335 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00017787207616493106, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311086472123861 }, "reported": { "count": 12, "observable_mse": 0.00017787201795727015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311097647994757 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006851697621195759 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200543329119682, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094702053815126 }, "reported": { "count": 98, "observable_mse": 0.00010200551332673058, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094708573073149 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004050559842515715 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543937292415649e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058296654373407 }, "reported": { "count": 72, "observable_mse": 8.543945295969024e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058302707970142 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.0037697612537818566 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00012222511577419937, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060947067104279995 }, "reported": { "count": 14, "observable_mse": 0.00012222547957208008, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094712298363447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0034719076328323475 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.0001778121804818511, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006308719515800476 }, "reported": { "count": 12, "observable_mse": 0.0001778119767550379, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0063087306916713715 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006852692037580273 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067623384296894 }, "reported": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0023185078919724916 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.603723108535632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067622918635607 }, "reported": { "count": 70, "observable_mse": 8.603732567280531e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.00202869440547593 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010919466149061918, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067620124667883 }, "reported": { "count": 14, "observable_mse": 0.00010919448686763644, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028981346969819097 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017808005213737488, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067628972232342 }, "reported": { "count": 14, "observable_mse": 0.00017807971744332463, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.005216642454567437 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.0001069663412636146, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "reported": { "count": 84, "observable_mse": 0.00010696651588659734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00017277487015111977 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001894764427561313, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360026644082518e-16 }, "reported": { "count": 14, "observable_mse": 0.00018947666103485972, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360023997104558e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-10, "numeric_tolerance_ratio": 0.00021421976386698359 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 6.69803557684645e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705172687768936 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.005974748847249882 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 4.724697646452114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754040114581585 }, "reported": { "count": 72, "observable_mse": 4.724687460111454e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754054084420204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.007448969996752607 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 6.864315218990669e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 14, "observable_mse": 6.864342140033841e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705178275704384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6763806343078613e-08, "numeric_tolerance_ratio": 0.008962120593553839 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00018344067211728543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405925706028938 }, "reported": { "count": 12, "observable_mse": 0.00018344077398069203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405914530158043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.0060718911179741195 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869952917099, "saved_at_worst": 1.2869952917099, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.6967987208633615, "worst_tolerance_index": [ 4903, 3 ], "actual_at_worst": -0.0002655982971191406, "saved_at_worst": -0.00026488304138183594, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.00011185490438947454, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010564569383859634 }, "reported": { "count": 6272, "observable_mse": 0.000111854933493305, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010564569383859634 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8781890808513913e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 9.627988038118929e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010497652925550938 }, "reported": { "count": 4092, "observable_mse": 9.627990948501974e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010497653856873512 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004543556941289534 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.0001191547344205901, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010564569383859634 }, "reported": { "count": 896, "observable_mse": 0.00011915479262825102, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010564569383859634 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.752225566889378e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.0001563972036819905, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010775051079690456 }, "reported": { "count": 1284, "observable_mse": 0.0001563972473377362, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.010775051079690456 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.298349564511559e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2947795391082764, "saved_at_worst": 1.2947795391082764, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4723996082751296, "worst_tolerance_index": [ 844, 2 ], "actual_at_worst": -0.0007253140211105347, "saved_at_worst": -0.0007248073816299438, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00019038809114135802, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01795356720685959 }, "reported": { "count": 3136, "observable_mse": 0.00019038810569327325, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01795356720685959 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.428004024717845e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.00017765552911441773, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018069889396429062 }, "reported": { "count": 1582, "observable_mse": 0.00017765557277016342, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01806989125907421 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006635740523678787 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00019056950986851007, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017953569069504738 }, "reported": { "count": 448, "observable_mse": 0.0001905695244204253, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017953569069504738 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4279786025202036e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.00020852700981777161, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01778586208820343 }, "reported": { "count": 1106, "observable_mse": 0.00020852700981777161, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01778586022555828 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006703572011485321 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2878199815750122, "saved_at_worst": 1.2878199815750122, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.41482060580750596, "worst_tolerance_index": [ 3035, 3 ], "actual_at_worst": 0.003649771213531494, "saved_at_worst": 0.0036503374576568604, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00040838465793058276, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028023647144436836 }, "reported": { "count": 3136, "observable_mse": 0.0004083846288267523, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028023645281791687 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0004898649604547301 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.0003915227425750345, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02775302715599537 }, "reported": { "count": 1001, "observable_mse": 0.00039152277167886496, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02775302715599537 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8007281604630173e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0004061652871314436, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028023649007081985 }, "reported": { "count": 448, "observable_mse": 0.0004061653162352741, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028023649007081985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.7967872479718438e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.0004189791507087648, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028182994574308395 }, "reported": { "count": 1687, "observable_mse": 0.0004189791507087648, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028182994574308395 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.328358769416809, "saved_at_worst": 1.328358769416809, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.109476089477539e-07, "max_tolerance_ratio": 0.379329818060494, "worst_tolerance_index": [ 1250, 2 ], "actual_at_worst": -0.0006059706211090088, "saved_at_worst": -0.0006063729524612427, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0009452107478864491, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04636046662926674 }, "reported": { "count": 3136, "observable_mse": 0.0009452107478864491, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04636046290397644 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006609758164706417 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0008695956785231829, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04342574626207352 }, "reported": { "count": 407, "observable_mse": 0.000869595562107861, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04342574253678322 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006972837665095773 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0009436380350962281, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04636046662926674 }, "reported": { "count": 448, "observable_mse": 0.00094363815151155, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04636046662926674 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010637716654662544 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0009590117260813713, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046864788979291916 }, "reported": { "count": 2281, "observable_mse": 0.0009590117260813713, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046864788979291916 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011598924174904823 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010999653022736311, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010999666119460016, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00012954231484367922 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011603816412389278 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010919027408817783, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010919012856902555, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014394738889362153 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010782991535961628 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011893396003870293, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011893427290488034, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003091888621587732 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010790933854877949 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011383872333681211, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011383901437511668, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00028776244525315775 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03806070610880852 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00042644605855457485, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.000426446640631184, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005582693981920547 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0369470976293087 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0004731984226964414, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00047319853911176324, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011115546162157263 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.035910893231630325 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0003548956592567265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00035489487345330417, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.8580342233181e-10, "numeric_tolerance_ratio": 0.0007588714631438345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020458154380321503 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018904275202658027, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018904254829976708, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.00019994696482166673 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 6.998432908560917e-16 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.00634012371301651 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00012711752788163722, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00012711742601823062, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.0001005847985299984 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008087845519185066 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.686173143563792e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.68617750913836e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3236944961253016e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 6.698037759633735e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.059283086545828e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012188127264380455 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00014413957251235843, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00014413951430469751, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.738058002000685e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252827632939443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.5279510989785194e-10, "numeric_tolerance_ratio": 0.00015124442457133805 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011474297207314521, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011474302300484851, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.035392711752096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010750066576292738, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010750071669463068, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0390007111411554e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010249437036691234, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6010698552450364e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001481530343880877, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00014815300528425723, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8678943293010077e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 0.00010700838174670935, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 0.00010700869461288676, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003095536838477713 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 6.998753087827936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 6.998747994657606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.057772256490849e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.253204239532351e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.253222429426387e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-10, "numeric_tolerance_ratio": 0.0001805890875343717 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002318347105756402, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00023183472512755543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.422219535332207e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.00010696651588659734, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 1.5491703379666433e-05, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 1.5491703379666433e-05 }, "gradients": { "norms": { "prediction": 0.0003186509129591286, "weighted_latent": 2.6831793128829187e-18, "total": 0.0003186509129591286 }, "prediction_weighted_latent_dot": 7.71858239031985e-23, "prediction_weighted_latent_cosine": 0.09027607741696084, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "e10d1479ab2365c43ffe7e27c9b1efa8384babeabb8f6328e4de58db8d11fd16" } ], "final_losses": { "prediction_mse": 0.00010696651588659734, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 1.5491703379666433e-05, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 1.5491703379666433e-05 }, "training_seconds": 4.3000036384910345e-05, "evaluation_seconds": 0.6642293999902904 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 3, "arm": "native", "status": "passed", "files": { "seed-3-native/result.json": "c9dc68011f01b4a7571815fef2f6ed43b2845347c9e56f0a09546724685fa851", "seed-3-native/checkpoint.json": "943e22bc3deb4e7c773e1efe266351336849e74d84ca86125d1aef1f2a7f1bf4", "inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "seed-3-native/diagnostics-arrays.npz": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 5.811452865600586e-07, "max_tolerance_ratio": 0.3990840958602454, "worst_tolerance_index": [ 60, 2 ], "actual_at_worst": 0.004562556743621826, "saved_at_worst": 0.004561975598335266, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 0.00010696637036744505, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 0.00010696651588659734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014397905845926648 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001894764427561313, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 0.00018947666103485972, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-10, "numeric_tolerance_ratio": 0.00021421976386698359 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2893109321594238, "saved_at_worst": 1.2893109321594238, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.3867421263170503, "worst_tolerance_index": [ 77, 11 ], "actual_at_worst": 0.0007888078689575195, "saved_at_worst": 0.0007883906364440918, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.0016188963782042265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16294221580028534 }, "reported": { "count": 784, "observable_mse": 0.0016188964946195483, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16294220089912415 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0008616266658095435 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 0.0017986265011131763, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.17537100613117218 }, "reported": { "count": 580, "observable_mse": 0.0017986266175284982, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.17537100613117218 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 9.866853626335093e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.0018708795541897416, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16294221580028534 }, "reported": { "count": 112, "observable_mse": 0.001870880019851029, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16294221580028534 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00039227191794461667 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 0.00017905373533722013, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "reported": { "count": 92, "observable_mse": 0.00017905370623338968, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8591882208963364e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.000102343074104283, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "reported": { "count": 98, "observable_mse": 0.00010234319051960483, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001757528884955003 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.67251874296926e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897133611142635 }, "reported": { "count": 70, "observable_mse": 8.67253402248025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0020504503657808367 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010630184988258407, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897137802094221 }, "reported": { "count": 14, "observable_mse": 0.00010630159522406757, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002343372258461114 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017647375352680683, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897135008126497 }, "reported": { "count": 14, "observable_mse": 0.00017647400090936571, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897139199078083 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002636292941319088 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252822539769113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.737066673347726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 70, "observable_mse": 8.737093594390899e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010268489859299734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 14, "observable_mse": 0.0001026846730383113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.255546860396862e-10, "numeric_tolerance_ratio": 0.00022326212619665206 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017815936007536948, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "reported": { "count": 14, "observable_mse": 0.00017815965111367404, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002859439373556026 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001030568455462344, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580707758665 }, "reported": { "count": 98, "observable_mse": 0.00010305678733857349, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00579952893372632 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.592774247517809e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580242097378 }, "reported": { "count": 70, "observable_mse": 8.592770609539002e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585904866456985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.006379481642109011 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001099406072171405, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585797764360905 }, "reported": { "count": 14, "observable_mse": 0.00010994031617883593, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585867613554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.004349647583201486 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00018181865743827075, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058583967387676 }, "reported": { "count": 14, "observable_mse": 0.00018181874474976212, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003769693806922108 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200838732998818, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097275298088789 }, "reported": { "count": 98, "observable_mse": 0.00010200848191743717, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097279489040375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002603515450311345 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543613512301818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060906685888767 }, "reported": { "count": 72, "observable_mse": 8.543617150280625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060910876840353 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026094108969952367 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001222111313836649, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00609727343544364 }, "reported": { "count": 14, "observable_mse": 0.00012221158249303699, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060972776263952255 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026035157515688335 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00017787207616493106, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311086472123861 }, "reported": { "count": 12, "observable_mse": 0.00017787201795727015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311097647994757 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006851697621195759 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200543329119682, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094702053815126 }, "reported": { "count": 98, "observable_mse": 0.00010200551332673058, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094708573073149 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004050559842515715 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543937292415649e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058296654373407 }, "reported": { "count": 72, "observable_mse": 8.543945295969024e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058302707970142 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.0037697612537818566 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00012222511577419937, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060947067104279995 }, "reported": { "count": 14, "observable_mse": 0.00012222547957208008, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094712298363447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0034719076328323475 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.0001778121804818511, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006308719515800476 }, "reported": { "count": 12, "observable_mse": 0.0001778119767550379, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0063087306916713715 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006852692037580273 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067623384296894 }, "reported": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0023185078919724916 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.603723108535632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067622918635607 }, "reported": { "count": 70, "observable_mse": 8.603732567280531e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.00202869440547593 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010919466149061918, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067620124667883 }, "reported": { "count": 14, "observable_mse": 0.00010919448686763644, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028981346969819097 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017808005213737488, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067628972232342 }, "reported": { "count": 14, "observable_mse": 0.00017807971744332463, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.005216642454567437 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.012269754894077778, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055108308792114 }, "reported": { "count": 98, "observable_mse": 0.012269754894077778, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055108308792114 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.011943741701543331, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055108308792114 }, "reported": { "count": 84, "observable_mse": 0.011943741701543331, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055111289024353 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0006333493125675963 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.014225834980607033, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055111289024353 }, "reported": { "count": 14, "observable_mse": 0.014225835911929607, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.46055108308792114 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.000633349352680734 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 6.69803557684645e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705172687768936 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.005974748847249882 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 4.724697646452114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754040114581585 }, "reported": { "count": 72, "observable_mse": 4.724687460111454e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754054084420204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.007448969996752607 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 6.864315218990669e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 14, "observable_mse": 6.864342140033841e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705178275704384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6763806343078613e-08, "numeric_tolerance_ratio": 0.008962120593553839 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00018344067211728543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405925706028938 }, "reported": { "count": 12, "observable_mse": 0.00018344077398069203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405914530158043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.0060718911179741195 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869952917099, "saved_at_worst": 1.2869952917099, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.47687637075463457, "worst_tolerance_index": [ 219, 2 ], "actual_at_worst": -0.0012496113777160645, "saved_at_worst": -0.001249074935913086, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.00640147877857089, "exact_state_accuracy": 0.9738520408163265, "per_field_accuracy": { "x": 0.9805484693877551, "y": 0.9950573979591837, "h": 0.9956951530612245 }, "normalized_latent_rmse": 0.2254096269607544 }, "reported": { "count": 6272, "observable_mse": 0.006401479244232178, "exact_state_accuracy": 0.9738520383834839, "per_field_accuracy": { "x": 0.9805484414100647, "y": 0.9950574040412903, "h": 0.9956951141357422 }, "normalized_latent_rmse": 0.2254096269607544 }, "differences": [], "max_errors": { "accuracy_abs": 3.892548228190407e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002839142008922737 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.007140567060559988, "exact_state_accuracy": 0.9701857282502444, "per_field_accuracy": { "x": 0.9782502443792767, "y": 0.9943792766373412, "h": 0.9946236559139785 }, "normalized_latent_rmse": 0.23947079479694366 }, "reported": { "count": 4092, "observable_mse": 0.007140567060559988, "exact_state_accuracy": 0.9701857566833496, "per_field_accuracy": { "x": 0.97825026512146, "y": 0.9943792819976807, "h": 0.9946236610412598 }, "normalized_latent_rmse": 0.23947079479694366 }, "differences": [], "max_errors": { "accuracy_abs": 2.843310520361797e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.006372062023729086, "exact_state_accuracy": 0.9720982142857143, "per_field_accuracy": { "x": 0.9799107142857143, "y": 0.9944196428571429, "h": 0.9944196428571429 }, "normalized_latent_rmse": 0.2254096269607544 }, "reported": { "count": 896, "observable_mse": 0.006372062489390373, "exact_state_accuracy": 0.9720982313156128, "per_field_accuracy": { "x": 0.9799107313156128, "y": 0.9944196939468384, "h": 0.9944196939468384 }, "normalized_latent_rmse": 0.2254096269607544 }, "differences": [], "max_errors": { "accuracy_abs": 5.1089695474182406e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028442432809519443 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.0040665967389941216, "exact_state_accuracy": 0.9867601246105919, "per_field_accuracy": { "x": 0.9883177570093458, "y": 0.9976635514018691, "h": 1.0 }, "normalized_latent_rmse": 0.17313982546329498 }, "reported": { "count": 1284, "observable_mse": 0.004066596273332834, "exact_state_accuracy": 0.9867600798606873, "per_field_accuracy": { "x": 0.9883177280426025, "y": 0.9976634979248047, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.1731398105621338 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0008136494816779414 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.4800254106521606, "saved_at_worst": 1.4800254106521606, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.364418029785156e-07, "max_tolerance_ratio": 0.4255259867803609, "worst_tolerance_index": [ 2845, 7 ], "actual_at_worst": -0.00138140469789505, "saved_at_worst": -0.00138092041015625, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.01766131818294525, "exact_state_accuracy": 0.9158163265306123, "per_field_accuracy": { "x": 0.9371811224489796, "y": 0.9620535714285714, "h": 1.0 }, "normalized_latent_rmse": 0.30599245429039 }, "reported": { "count": 3136, "observable_mse": 0.017661316320300102, "exact_state_accuracy": 0.9158163070678711, "per_field_accuracy": { "x": 0.9371811151504517, "y": 0.9620535373687744, "h": 1.0 }, "normalized_latent_rmse": 0.30599245429039 }, "differences": [], "max_errors": { "accuracy_abs": 3.405979698278827e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006733754560566584 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.019254149869084358, "exact_state_accuracy": 0.9064475347661188, "per_field_accuracy": { "x": 0.9292035398230089, "y": 0.95448798988622, "h": 1.0 }, "normalized_latent_rmse": 0.31791701912879944 }, "reported": { "count": 1582, "observable_mse": 0.019254149869084358, "exact_state_accuracy": 0.9064475893974304, "per_field_accuracy": { "x": 0.9292035698890686, "y": 0.9544880390167236, "h": 1.0 }, "normalized_latent_rmse": 0.31791701912879944 }, "differences": [], "max_errors": { "accuracy_abs": 5.463131158567336e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.017359385266900063, "exact_state_accuracy": 0.9151785714285714, "per_field_accuracy": { "x": 0.9375, "y": 0.9620535714285714, "h": 1.0 }, "normalized_latent_rmse": 0.30599245429039 }, "reported": { "count": 448, "observable_mse": 0.017359385266900063, "exact_state_accuracy": 0.9151785969734192, "per_field_accuracy": { "x": 0.9375000596046448, "y": 0.9620535969734192, "h": 1.0 }, "normalized_latent_rmse": 0.30599245429039 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.01550526637583971, "exact_state_accuracy": 0.9294755877034359, "per_field_accuracy": { "x": 0.9484629294755877, "y": 0.972875226039783, "h": 1.0 }, "normalized_latent_rmse": 0.28807908296585083 }, "reported": { "count": 1106, "observable_mse": 0.01550526823848486, "exact_state_accuracy": 0.9294756054878235, "per_field_accuracy": { "x": 0.948462963104248, "y": 0.9728752374649048, "h": 1.0 }, "normalized_latent_rmse": 0.28807908296585083 }, "differences": [], "max_errors": { "accuracy_abs": 3.36286603008773e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0007302982002833496 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.515133023262024, "saved_at_worst": 1.515133023262024, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.35522893439134723, "worst_tolerance_index": [ 716, 3 ], "actual_at_worst": 6.788969039916992e-05, "saved_at_worst": 6.753206253051758e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.044902168214321136, "exact_state_accuracy": 0.7213010204081632, "per_field_accuracy": { "x": 0.8118622448979592, "y": 0.9298469387755102, "h": 0.9371811224489796 }, "normalized_latent_rmse": 0.4604598581790924 }, "reported": { "count": 3136, "observable_mse": 0.044902168214321136, "exact_state_accuracy": 0.7213010191917419, "per_field_accuracy": { "x": 0.8118622303009033, "y": 0.9298469424247742, "h": 0.9371811151504517 }, "normalized_latent_rmse": 0.4604598581790924 }, "differences": [], "max_errors": { "accuracy_abs": 1.4597055897347389e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.048988405615091324, "exact_state_accuracy": 0.6993006993006993, "per_field_accuracy": { "x": 0.8001998001998002, "y": 0.9430569430569431, "h": 0.9140859140859141 }, "normalized_latent_rmse": 0.4937666654586792 }, "reported": { "count": 1001, "observable_mse": 0.048988401889801025, "exact_state_accuracy": 0.6993007063865662, "per_field_accuracy": { "x": 0.8001998066902161, "y": 0.9430569410324097, "h": 0.914085865020752 }, "normalized_latent_rmse": 0.4937666654586792 }, "differences": [], "max_errors": { "accuracy_abs": 4.906516215541501e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006315292801831286 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.044932298362255096, "exact_state_accuracy": 0.7254464285714286, "per_field_accuracy": { "x": 0.8214285714285714, "y": 0.9330357142857143, "h": 0.9375 }, "normalized_latent_rmse": 0.4604598879814148 }, "reported": { "count": 448, "observable_mse": 0.044932298362255096, "exact_state_accuracy": 0.7254464626312256, "per_field_accuracy": { "x": 0.8214285969734192, "y": 0.9330357313156128, "h": 0.9375000596046448 }, "normalized_latent_rmse": 0.4604598581790924 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0006334721628120355 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.04246954992413521, "exact_state_accuracy": 0.7332542975696502, "per_field_accuracy": { "x": 0.8162418494368702, "y": 0.921161825726141, "h": 0.950800237107291 }, "normalized_latent_rmse": 0.43950536847114563 }, "reported": { "count": 1687, "observable_mse": 0.04246955364942551, "exact_state_accuracy": 0.7332543134689331, "per_field_accuracy": { "x": 0.8162418603897095, "y": 0.9211618900299072, "h": 0.9508002996444702 }, "normalized_latent_rmse": 0.43950533866882324 }, "differences": [], "max_errors": { "accuracy_abs": 6.430376620070177e-08, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0007099908498083255 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.4931187629699707, "saved_at_worst": 1.4931187629699707, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.38143623930895526, "worst_tolerance_index": [ 2582, 8 ], "actual_at_worst": -0.0009388774633407593, "saved_at_worst": -0.0009384602308273315, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.08657023310661316, "exact_state_accuracy": 0.5446428571428571, "per_field_accuracy": { "x": 0.7085459183673469, "y": 0.7911352040816326, "h": 0.8590561224489796 }, "normalized_latent_rmse": 0.5297196507453918 }, "reported": { "count": 3136, "observable_mse": 0.08657022565603256, "exact_state_accuracy": 0.5446428656578064, "per_field_accuracy": { "x": 0.7085459232330322, "y": 0.7911351919174194, "h": 0.8590561151504517 }, "normalized_latent_rmse": 0.5297196507453918 }, "differences": [], "max_errors": { "accuracy_abs": 1.216421319227834e-08, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0007715194353445527 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.11863617599010468, "exact_state_accuracy": 0.40540540540540543, "per_field_accuracy": { "x": 0.6486486486486487, "y": 0.687960687960688, "h": 0.7985257985257985 }, "normalized_latent_rmse": 0.6136864423751831 }, "reported": { "count": 407, "observable_mse": 0.11863616108894348, "exact_state_accuracy": 0.4054054021835327, "per_field_accuracy": { "x": 0.6486486196517944, "y": 0.687960684299469, "h": 0.7985258102416992 }, "normalized_latent_rmse": 0.6136864423751831 }, "differences": [], "max_errors": { "accuracy_abs": 2.8996854251062132e-08, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.001158395980391896 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.08620558679103851, "exact_state_accuracy": 0.5379464285714286, "per_field_accuracy": { "x": 0.703125, "y": 0.7946428571428571, "h": 0.8616071428571429 }, "normalized_latent_rmse": 0.5297196507453918 }, "reported": { "count": 448, "observable_mse": 0.08620557934045792, "exact_state_accuracy": 0.5379464626312256, "per_field_accuracy": { "x": 0.7031250596046448, "y": 0.7946428656578064, "h": 0.8616071939468384 }, "normalized_latent_rmse": 0.5297196507453918 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0007744437118929744 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.08092029392719269, "exact_state_accuracy": 0.5708022797018851, "per_field_accuracy": { "x": 0.7202981148619026, "y": 0.8088557650153442, "h": 0.8693555458132398 }, "normalized_latent_rmse": 0.5132953524589539 }, "reported": { "count": 2281, "observable_mse": 0.08092029392719269, "exact_state_accuracy": 0.5708022713661194, "per_field_accuracy": { "x": 0.7202981114387512, "y": 0.8088557720184326, "h": 0.8693555593490601 }, "normalized_latent_rmse": 0.5132953524589539 }, "differences": [], "max_errors": { "accuracy_abs": 1.353582024243849e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011598924174904823 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010999653022736311, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010999666119460016, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00012954231484367922 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011603816412389278 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010919027408817783, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010919012856902555, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014394738889362153 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010782991535961628 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011893396003870293, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011893427290488034, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003091888621587732 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010790933854877949 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011383872333681211, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011383901437511668, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00028776244525315775 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03806070610880852 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00042644605855457485, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.000426446640631184, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005582693981920547 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0369470976293087 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0004731984226964414, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00047319853911176324, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011115546162157263 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.035910893231630325 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0003548956592567265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00035489487345330417, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.8580342233181e-10, "numeric_tolerance_ratio": 0.0007588714631438345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020458154380321503 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018904275202658027, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018904254829976708, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.00019994696482166673 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 0.6938775510204082, "normalized_latent_relation_rmse": 0.5547308325767517 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.04983695596456528, "exact_state_accuracy": 0.6938775510204082, "per_field_accuracy": { "x": 0.9183673469387755, "y": 1.0, "h": 0.7244897959183674 } }, "reported": { "count": 98, "observable_mse": 0.04983697086572647, "exact_state_accuracy": 0.6938775181770325, "per_field_accuracy": { "x": 0.918367326259613, "y": 1.0, "h": 0.7244898080825806 } }, "differences": [], "max_errors": { "accuracy_abs": 3.28433756857649e-08, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.002490293371849605 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 0.20408163265306123, "normalized_latent_relation_rmse": 0.6198129653930664 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.11789534240961075, "exact_state_accuracy": 0.20408163265306123, "per_field_accuracy": { "x": 0.2755102040816326, "y": 0.9795918367346939, "h": 0.8163265306122449 } }, "reported": { "count": 98, "observable_mse": 0.11789533495903015, "exact_state_accuracy": 0.20408162474632263, "per_field_accuracy": { "x": 0.27551019191741943, "y": 0.9795918464660645, "h": 0.8163264989852905 } }, "differences": [], "max_errors": { "accuracy_abs": 3.1626954388741524e-08, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0005825529601459301 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008087845519185066 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.686173143563792e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.68617750913836e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3236944961253016e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 6.698037759633735e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.059283086545828e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012188127264380455 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00014413957251235843, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00014413951430469751, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.738058002000685e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252827632939443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.5279510989785194e-10, "numeric_tolerance_ratio": 0.00015124442457133805 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011474297207314521, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011474302300484851, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.035392711752096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010750066576292738, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010750071669463068, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0390007111411554e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 0.7653061224489796 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0509251169860363, "exact_state_accuracy": 0.6224489795918368, "per_field_accuracy": { "x": 0.6428571428571429, "y": 0.9693877551020408, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0509251169860363, "exact_state_accuracy": 0.6224489808082581, "per_field_accuracy": { "x": 0.6428571343421936, "y": 0.9693877100944519, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 4.5007588878043236e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.012192781083285809, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.01219277922064066, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0008393023382572028 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 0.00010700838174670935, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 0.00010700869461288676, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003095536838477713 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 6.998753087827936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 6.998747994657606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.057772256490849e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.253204239532351e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.253222429426387e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-10, "numeric_tolerance_ratio": 0.0001805890875343717 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002318347105756402, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00023183472512755543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.422219535332207e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 774, "error_count": 206, "per_field_correct_counts": { "x": 811, "y": 889, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 210, "error_count": 476, "per_field_correct_counts": { "x": 450, "y": 482, "h": 515 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 212, "error_count": 278, "per_field_correct_counts": { "x": 269, "y": 401, "h": 453 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 0, "error_count": 98, "per_field_correct_counts": { "x": 98, "y": 21, "h": 49 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 4, "error_count": 192, "per_field_correct_counts": { "x": 14, "y": 143, "h": 104 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 0, "error_count": 98, "per_field_correct_counts": { "x": 0, "y": 21, "h": 54 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 18, "error_count": 80, "per_field_correct_counts": { "x": 90, "y": 34, "h": 49 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 1708, "accuracy": 0.5446428571428571 }, "numpy": { "cases": 3136, "correct": 1708, "accuracy": 0.5446428571428571 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.011943741701543331, "latent_mse": 0.212107315659523, "scaled_prediction": 0.0017297833692282438, "scaled_weighted_latent": 0.0030718990601599216, "total": 0.004801682196557522 }, "gradients": { "norms": { "prediction": 0.0034363465383648872, "weighted_latent": 0.004243341740220785, "total": 0.0022537540644407272 }, "prediction_weighted_latent_dot": -1.2367510862532072e-05, "prediction_weighted_latent_cosine": -0.8481591220444237, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "943e22bc3deb4e7c773e1efe266351336849e74d84ca86125d1aef1f2a7f1bf4" } ], "final_losses": { "prediction_mse": 0.011943741701543331, "latent_mse": 0.212107315659523, "scaled_prediction": 0.0017297833692282438, "scaled_weighted_latent": 0.0030718990601599216, "total": 0.004801682196557522 }, "training_seconds": 4.0899962186813354e-05, "evaluation_seconds": 0.5442906999960542 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 3, "arm": "prediction_latent", "status": "passed", "files": { "seed-3-prediction_latent/result.json": "b2c4aa384a0aeec0941d8e7ecbcf32ca812b4066261f9896fde0676749ca3402", "seed-3-prediction_latent/checkpoint.json": "fe943a3c3a39ef1a0d8246e36a9c3703472e67a373e38cf26ebb5945840de54e", "inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "seed-3-prediction_latent/diagnostics-arrays.npz": "c76bf30e8ceb95e67340c590a1f774ac16bb2f18e84ab8870d9b8e3fa8d3fdc0" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 5.811452865600586e-07, "max_tolerance_ratio": 0.3990840958602454, "worst_tolerance_index": [ 60, 2 ], "actual_at_worst": 0.004562556743621826, "saved_at_worst": 0.004561975598335266, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 0.00010696637036744505, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 0.00010696651588659734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014397905845926648 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001894764427561313, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 0.00018947666103485972, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-10, "numeric_tolerance_ratio": 0.00021421976386698359 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2893109321594238, "saved_at_worst": 1.2893109321594238, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.3867421263170503, "worst_tolerance_index": [ 77, 11 ], "actual_at_worst": 0.0007888078689575195, "saved_at_worst": 0.0007883906364440918, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.0012843458680436015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.11893079429864883 }, "reported": { "count": 784, "observable_mse": 0.0012843457516282797, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.11893079429864883 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010316532689556824 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 0.0014315596781671047, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1279904842376709 }, "reported": { "count": 580, "observable_mse": 0.0014315597945824265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.1279904842376709 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010183677811150988 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.001429907395504415, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.11893079429864883 }, "reported": { "count": 112, "observable_mse": 0.001429907395504415, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.11893079429864883 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 0.00017905373533722013, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "reported": { "count": 92, "observable_mse": 0.00017905370623338968, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8591882208963364e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.000102343074104283, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "reported": { "count": 98, "observable_mse": 0.00010234319051960483, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001757528884955003 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.67251874296926e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897133611142635 }, "reported": { "count": 70, "observable_mse": 8.67253402248025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0020504503657808367 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010630184988258407, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897137802094221 }, "reported": { "count": 14, "observable_mse": 0.00010630159522406757, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002343372258461114 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017647375352680683, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897135008126497 }, "reported": { "count": 14, "observable_mse": 0.00017647400090936571, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897139199078083 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002636292941319088 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252822539769113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.737066673347726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 70, "observable_mse": 8.737093594390899e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010268489859299734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 14, "observable_mse": 0.0001026846730383113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.255546860396862e-10, "numeric_tolerance_ratio": 0.00022326212619665206 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017815936007536948, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "reported": { "count": 14, "observable_mse": 0.00017815965111367404, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002859439373556026 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001030568455462344, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580707758665 }, "reported": { "count": 98, "observable_mse": 0.00010305678733857349, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00579952893372632 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.592774247517809e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580242097378 }, "reported": { "count": 70, "observable_mse": 8.592770609539002e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585904866456985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.006379481642109011 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001099406072171405, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585797764360905 }, "reported": { "count": 14, "observable_mse": 0.00010994031617883593, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585867613554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.004349647583201486 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00018181865743827075, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058583967387676 }, "reported": { "count": 14, "observable_mse": 0.00018181874474976212, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003769693806922108 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200838732998818, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097275298088789 }, "reported": { "count": 98, "observable_mse": 0.00010200848191743717, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097279489040375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002603515450311345 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543613512301818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060906685888767 }, "reported": { "count": 72, "observable_mse": 8.543617150280625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060910876840353 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026094108969952367 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001222111313836649, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00609727343544364 }, "reported": { "count": 14, "observable_mse": 0.00012221158249303699, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060972776263952255 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026035157515688335 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00017787207616493106, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311086472123861 }, "reported": { "count": 12, "observable_mse": 0.00017787201795727015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311097647994757 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006851697621195759 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200543329119682, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094702053815126 }, "reported": { "count": 98, "observable_mse": 0.00010200551332673058, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094708573073149 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004050559842515715 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543937292415649e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058296654373407 }, "reported": { "count": 72, "observable_mse": 8.543945295969024e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058302707970142 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.0037697612537818566 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00012222511577419937, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060947067104279995 }, "reported": { "count": 14, "observable_mse": 0.00012222547957208008, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094712298363447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0034719076328323475 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.0001778121804818511, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006308719515800476 }, "reported": { "count": 12, "observable_mse": 0.0001778119767550379, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0063087306916713715 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006852692037580273 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067623384296894 }, "reported": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0023185078919724916 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.603723108535632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067622918635607 }, "reported": { "count": 70, "observable_mse": 8.603732567280531e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.00202869440547593 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010919466149061918, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067620124667883 }, "reported": { "count": 14, "observable_mse": 0.00010919448686763644, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028981346969819097 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017808005213737488, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067628972232342 }, "reported": { "count": 14, "observable_mse": 0.00017807971744332463, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.005216642454567437 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.009593349881470203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594977855682373 }, "reported": { "count": 98, "observable_mse": 0.009593350812792778, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594974875450134 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9802322387695312e-08, "numeric_tolerance_ratio": 0.0008614639118828826 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.009409232065081596, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594974875450134 }, "reported": { "count": 84, "observable_mse": 0.009409232996404171, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594974875450134 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004798348161351965 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.010698058642446995, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594974875450134 }, "reported": { "count": 14, "observable_mse": 0.01069805957376957, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.33594974875450134 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00044995646635191525 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 6.69803557684645e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705172687768936 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.005974748847249882 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 4.724697646452114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754040114581585 }, "reported": { "count": 72, "observable_mse": 4.724687460111454e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754054084420204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.007448969996752607 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 6.864315218990669e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 14, "observable_mse": 6.864342140033841e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705178275704384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6763806343078613e-08, "numeric_tolerance_ratio": 0.008962120593553839 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00018344067211728543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405925706028938 }, "reported": { "count": 12, "observable_mse": 0.00018344077398069203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405914530158043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.0060718911179741195 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869952917099, "saved_at_worst": 1.2869952917099, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.5021484272770335, "worst_tolerance_index": [ 5148, 2 ], "actual_at_worst": -8.893013000488281e-05, "saved_at_worst": -8.943676948547363e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.004743542522192001, "exact_state_accuracy": 0.9770408163265306, "per_field_accuracy": { "x": 0.9858099489795918, "y": 0.9912308673469388, "h": 1.0 }, "normalized_latent_rmse": 0.1644071638584137 }, "reported": { "count": 6272, "observable_mse": 0.004743542056530714, "exact_state_accuracy": 0.9770407676696777, "per_field_accuracy": { "x": 0.9858099222183228, "y": 0.991230845451355, "h": 1.0 }, "normalized_latent_rmse": 0.1644071638584137 }, "differences": [], "max_errors": { "accuracy_abs": 4.865685288013566e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00031584085121626025 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.005354098044335842, "exact_state_accuracy": 0.9733626588465298, "per_field_accuracy": { "x": 0.9838709677419355, "y": 0.9894916911045943, "h": 1.0 }, "normalized_latent_rmse": 0.17495006322860718 }, "reported": { "count": 4092, "observable_mse": 0.005354098044335842, "exact_state_accuracy": 0.9733626842498779, "per_field_accuracy": { "x": 0.9838709831237793, "y": 0.9894917011260986, "h": 1.0 }, "normalized_latent_rmse": 0.17495006322860718 }, "differences": [], "max_errors": { "accuracy_abs": 2.540334809175704e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.004613194614648819, "exact_state_accuracy": 0.9776785714285714, "per_field_accuracy": { "x": 0.9866071428571429, "y": 0.9910714285714286, "h": 1.0 }, "normalized_latent_rmse": 0.1644071638584137 }, "reported": { "count": 896, "observable_mse": 0.004613194614648819, "exact_state_accuracy": 0.9776785969734192, "per_field_accuracy": { "x": 0.9866071939468384, "y": 0.9910714626312256, "h": 1.0 }, "normalized_latent_rmse": 0.1644071638584137 }, "differences": [], "max_errors": { "accuracy_abs": 5.1089695474182406e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.002888710703700781, "exact_state_accuracy": 0.9883177570093458, "per_field_accuracy": { "x": 0.9914330218068536, "y": 0.9968847352024922, "h": 1.0 }, "normalized_latent_rmse": 0.12501011788845062 }, "reported": { "count": 1284, "observable_mse": 0.0028887102380394936, "exact_state_accuracy": 0.9883177280426025, "per_field_accuracy": { "x": 0.9914329648017883, "y": 0.9968847036361694, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.12501011788845062 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00036129393764582855 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.280820369720459, "saved_at_worst": 1.280820369720459, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.5498198938627676, "worst_tolerance_index": [ 2107, 2 ], "actual_at_worst": 2.8252601623535156e-05, "saved_at_worst": 2.7701258659362793e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.01111413910984993, "exact_state_accuracy": 0.9346301020408163, "per_field_accuracy": { "x": 0.9764030612244898, "y": 0.9566326530612245, "h": 1.0 }, "normalized_latent_rmse": 0.21937502920627594 }, "reported": { "count": 3136, "observable_mse": 0.01111413724720478, "exact_state_accuracy": 0.9346300959587097, "per_field_accuracy": { "x": 0.9764030575752258, "y": 0.9566326141357422, "h": 1.0 }, "normalized_latent_rmse": 0.21937502920627594 }, "differences": [], "max_errors": { "accuracy_abs": 3.892548228190407e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0008821791425446691 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.012310564517974854, "exact_state_accuracy": 0.9254108723135271, "per_field_accuracy": { "x": 0.9740834386852086, "y": 0.9487989886219975, "h": 1.0 }, "normalized_latent_rmse": 0.2291337251663208 }, "reported": { "count": 1582, "observable_mse": 0.012310565449297428, "exact_state_accuracy": 0.9254109263420105, "per_field_accuracy": { "x": 0.9740834832191467, "y": 0.9487990140914917, "h": 1.0 }, "normalized_latent_rmse": 0.2291337251663208 }, "differences": [], "max_errors": { "accuracy_abs": 5.4028483353896206e-08, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00041743566595476244 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.011239134706556797, "exact_state_accuracy": 0.9397321428571429, "per_field_accuracy": { "x": 0.9776785714285714, "y": 0.9598214285714286, "h": 1.0 }, "normalized_latent_rmse": 0.21937502920627594 }, "reported": { "count": 448, "observable_mse": 0.011239135637879372, "exact_state_accuracy": 0.9397321939468384, "per_field_accuracy": { "x": 0.9776785969734192, "y": 0.9598214626312256, "h": 1.0 }, "normalized_latent_rmse": 0.21937502920627594 }, "differences": [], "max_errors": { "accuracy_abs": 5.1089695474182406e-08, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00043849363293037785 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.009352161549031734, "exact_state_accuracy": 0.945750452079566, "per_field_accuracy": { "x": 0.9792043399638336, "y": 0.9665461121157324, "h": 1.0 }, "normalized_latent_rmse": 0.2046089768409729 }, "reported": { "count": 1106, "observable_mse": 0.009352161549031734, "exact_state_accuracy": 0.9457504749298096, "per_field_accuracy": { "x": 0.9792043566703796, "y": 0.9665461182594299, "h": 1.0 }, "normalized_latent_rmse": 0.2046089917421341 }, "differences": [], "max_errors": { "accuracy_abs": 2.2850243586169938e-08, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.000694340021491379 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.363230586051941, "saved_at_worst": 1.363230586051941, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.556510925292969e-07, "max_tolerance_ratio": 0.43877816078235904, "worst_tolerance_index": [ 174, 6 ], "actual_at_worst": -0.000697113573551178, "saved_at_worst": -0.0006975829601287842, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.031687669456005096, "exact_state_accuracy": 0.7955994897959183, "per_field_accuracy": { "x": 0.8931760204081632, "y": 0.9024234693877551, "h": 1.0 }, "normalized_latent_rmse": 0.3262229263782501 }, "reported": { "count": 3136, "observable_mse": 0.031687669456005096, "exact_state_accuracy": 0.7955994606018066, "per_field_accuracy": { "x": 0.8931760191917419, "y": 0.9024234414100647, "h": 1.0 }, "normalized_latent_rmse": 0.3262229263782501 }, "differences": [], "max_errors": { "accuracy_abs": 2.9194111683672475e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.03607439249753952, "exact_state_accuracy": 0.7812187812187812, "per_field_accuracy": { "x": 0.8851148851148851, "y": 0.8961038961038961, "h": 1.0 }, "normalized_latent_rmse": 0.3467058539390564 }, "reported": { "count": 1001, "observable_mse": 0.03607438877224922, "exact_state_accuracy": 0.7812187671661377, "per_field_accuracy": { "x": 0.885114848613739, "y": 0.8961038589477539, "h": 1.0 }, "normalized_latent_rmse": 0.3467058539390564 }, "differences": [], "max_errors": { "accuracy_abs": 3.715614216304175e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0008085381917655889 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.03193138167262077, "exact_state_accuracy": 0.7924107142857143, "per_field_accuracy": { "x": 0.8950892857142857, "y": 0.8973214285714286, "h": 1.0 }, "normalized_latent_rmse": 0.3262229263782501 }, "reported": { "count": 448, "observable_mse": 0.03193138539791107, "exact_state_accuracy": 0.7924107313156128, "per_field_accuracy": { "x": 0.895089328289032, "y": 0.8973214626312256, "h": 1.0 }, "normalized_latent_rmse": 0.3262229263782501 }, "differences": [], "max_errors": { "accuracy_abs": 4.257474628399649e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.000888425284094596 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.029020041227340698, "exact_state_accuracy": 0.8049792531120332, "per_field_accuracy": { "x": 0.8974510966212211, "y": 0.9075281564908121, "h": 1.0 }, "normalized_latent_rmse": 0.3134370446205139 }, "reported": { "count": 1687, "observable_mse": 0.029020043089985847, "exact_state_accuracy": 0.8049792647361755, "per_field_accuracy": { "x": 0.8974511027336121, "y": 0.9075281620025635, "h": 1.0 }, "normalized_latent_rmse": 0.3134370446205139 }, "differences": [], "max_errors": { "accuracy_abs": 1.1624142315369568e-08, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0004773559949524988 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.340502381324768, "saved_at_worst": 1.340502381324768, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.3502697441310343, "worst_tolerance_index": [ 161, 2 ], "actual_at_worst": 0.0006358623504638672, "saved_at_worst": 0.000635489821434021, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.046413373202085495, "exact_state_accuracy": 0.6910076530612245, "per_field_accuracy": { "x": 0.857780612244898, "y": 0.7991071428571429, "h": 0.9971301020408163 }, "normalized_latent_rmse": 0.3625972270965576 }, "reported": { "count": 3136, "observable_mse": 0.046413373202085495, "exact_state_accuracy": 0.6910076141357422, "per_field_accuracy": { "x": 0.8577805757522583, "y": 0.7991071343421936, "h": 0.9971300959587097 }, "normalized_latent_rmse": 0.3625972270965576 }, "differences": [], "max_errors": { "accuracy_abs": 3.892548228190407e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.06509196758270264, "exact_state_accuracy": 0.5454545454545454, "per_field_accuracy": { "x": 0.7936117936117936, "y": 0.687960687960688, "h": 0.995085995085995 }, "normalized_latent_rmse": 0.41973230242729187 }, "reported": { "count": 407, "observable_mse": 0.06509196013212204, "exact_state_accuracy": 0.5454545617103577, "per_field_accuracy": { "x": 0.7936117649078369, "y": 0.687960684299469, "h": 0.9950859546661377 }, "normalized_latent_rmse": 0.41973230242729187 }, "differences": [], "max_errors": { "accuracy_abs": 4.041985735003806e-08, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0009921941821487623 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.04654546454548836, "exact_state_accuracy": 0.6852678571428571, "per_field_accuracy": { "x": 0.8482142857142857, "y": 0.7991071428571429, "h": 1.0 }, "normalized_latent_rmse": 0.36259725689888 }, "reported": { "count": 448, "observable_mse": 0.04654546454548836, "exact_state_accuracy": 0.6852678656578064, "per_field_accuracy": { "x": 0.848214328289032, "y": 0.7991071939468384, "h": 1.0 }, "normalized_latent_rmse": 0.36259725689888 }, "differences": [], "max_errors": { "accuracy_abs": 5.1089695474182406e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.04305460304021835, "exact_state_accuracy": 0.7181060938185007, "per_field_accuracy": { "x": 0.8711091626479615, "y": 0.8189390618149934, "h": 0.9969311705392372 }, "normalized_latent_rmse": 0.3514273464679718 }, "reported": { "count": 2281, "observable_mse": 0.04305460304021835, "exact_state_accuracy": 0.7181060910224915, "per_field_accuracy": { "x": 0.8711091876029968, "y": 0.8189390897750854, "h": 0.9969311952590942 }, "normalized_latent_rmse": 0.3514273464679718 }, "differences": [], "max_errors": { "accuracy_abs": 2.7960092019441163e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011598924174904823 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010999653022736311, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010999666119460016, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00012954231484367922 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011603816412389278 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010919027408817783, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010919012856902555, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014394738889362153 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010782991535961628 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011893396003870293, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011893427290488034, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003091888621587732 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010790933854877949 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011383872333681211, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011383901437511668, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00028776244525315775 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03806070610880852 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00042644605855457485, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.000426446640631184, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005582693981920547 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0369470976293087 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0004731984226964414, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00047319853911176324, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011115546162157263 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.035910893231630325 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0003548956592567265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00035489487345330417, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.8580342233181e-10, "numeric_tolerance_ratio": 0.0007588714631438345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020458154380321503 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018904275202658027, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018904254829976708, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.00019994696482166673 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 0.8775510204081632, "normalized_latent_relation_rmse": 0.37731829285621643 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.02961549162864685, "exact_state_accuracy": 0.8775510204081632, "per_field_accuracy": { "x": 0.8775510204081632, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0296154897660017, "exact_state_accuracy": 0.8775510191917419, "per_field_accuracy": { "x": 0.8775510191917419, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 1.2164212970233734e-09, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.00047018102268408466 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 0.41836734693877553, "normalized_latent_relation_rmse": 0.4238298237323761 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.08056417107582092, "exact_state_accuracy": 0.41836734693877553, "per_field_accuracy": { "x": 0.41836734693877553, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.08056416362524033, "exact_state_accuracy": 0.41836732625961304, "per_field_accuracy": { "x": 0.41836732625961304, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 2.0679162493486558e-08, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.000822685298321172 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008087845519185066 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.686173143563792e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.68617750913836e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3236944961253016e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 6.698037759633735e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.059283086545828e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012188127264380455 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00014413957251235843, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00014413951430469751, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.738058002000685e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252827632939443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.5279510989785194e-10, "numeric_tolerance_ratio": 0.00015124442457133805 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011474297207314521, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011474302300484851, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.035392711752096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010750066576292738, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010750071669463068, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0390007111411554e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 0.7857142857142857 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.04079073667526245, "exact_state_accuracy": 0.6632653061224489, "per_field_accuracy": { "x": 0.6632653061224489, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.04079073294997215, "exact_state_accuracy": 0.6632652878761292, "per_field_accuracy": { "x": 0.6632652878761292, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 1.824631978841751e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0007334586610772579 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.009554969146847725, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00955496821552515, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00047625880254618854 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 0.00010700838174670935, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 0.00010700869461288676, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003095536838477713 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 6.998753087827936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 6.998747994657606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.057772256490849e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.253204239532351e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.253222429426387e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-10, "numeric_tolerance_ratio": 0.0001805890875343717 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002318347105756402, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00023183472512755543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.422219535332207e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 861, "error_count": 119, "per_field_correct_counts": { "x": 941, "y": 899, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 382, "error_count": 304, "per_field_correct_counts": { "x": 556, "y": 512, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 314, "error_count": 176, "per_field_correct_counts": { "x": 376, "y": 414, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 21, "error_count": 77, "per_field_correct_counts": { "x": 63, "y": 28, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 55, "error_count": 141, "per_field_correct_counts": { "x": 87, "y": 114, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 25, "error_count": 73, "per_field_correct_counts": { "x": 94, "y": 28, "h": 95 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 19, "error_count": 79, "per_field_correct_counts": { "x": 83, "y": 21, "h": 92 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 2167, "accuracy": 0.6910076530612245 }, "numpy": { "cases": 3136, "correct": 2167, "accuracy": 0.6910076530612245 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.011943741701543331, "latent_mse": 0.212107315659523, "scaled_prediction": 0.0017297833692282438, "scaled_weighted_latent": 0.0030718990601599216, "total": 0.004801682196557522 }, "gradients": { "norms": { "prediction": 0.0034363465383648872, "weighted_latent": 0.004243341740220785, "total": 0.0022537540644407272 }, "prediction_weighted_latent_dot": -1.2367510862532072e-05, "prediction_weighted_latent_cosine": -0.8481591220444237, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.012193200178444386, "latent_mse": 0.20949852466583252, "scaled_prediction": 0.0017659118166193366, "scaled_weighted_latent": 0.0030341167002916336, "total": 0.004800028633326292 }, "gradients": { "norms": { "prediction": 0.0036069219931960106, "weighted_latent": 0.004211698193103075, "total": 0.0025878201704472303 }, "prediction_weighted_latent_dot": -1.2025736396026332e-05, "prediction_weighted_latent_cosine": -0.791621694330808, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.03035777248442173 }, { "step": 100, "losses": { "prediction_mse": 0.012208578176796436, "latent_mse": 0.12798641622066498, "scaled_prediction": 0.0017681389581412077, "scaled_weighted_latent": 0.001853596419095993, "total": 0.0036217353772372007 }, "gradients": { "norms": { "prediction": 0.003790223505347967, "weighted_latent": 0.003106442280113697, "total": 0.0034755005035549402 }, "prediction_weighted_latent_dot": -5.968337063677609e-06, "prediction_weighted_latent_cosine": -0.5069034335374009, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.023319583386182785 }, { "step": 200, "losses": { "prediction_mse": 0.009409403428435326, "latent_mse": 0.1128612533211708, "scaled_prediction": 0.0013627412263303995, "scaled_weighted_latent": 0.0016345423646271229, "total": 0.0029972835909575224 }, "gradients": { "norms": { "prediction": 0.002912612399086356, "weighted_latent": 0.002912527183070779, "total": 1.4770042071177159e-05 }, "prediction_weighted_latent_dot": -8.482954399369191e-06, "prediction_weighted_latent_cosine": -0.999987223162017, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00015583900676574558 }, { "step": 300, "losses": { "prediction_mse": 0.009409047663211823, "latent_mse": 0.11286406964063644, "scaled_prediction": 0.0013626896543428302, "scaled_weighted_latent": 0.0016345831099897623, "total": 0.0029972726479172707 }, "gradients": { "norms": { "prediction": 0.0029125649016350508, "weighted_latent": 0.0029125751461833715, "total": 1.37335703698227e-07 }, "prediction_weighted_latent_dot": -8.483063538733404e-06, "prediction_weighted_latent_cosine": -0.9999999286325258, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.854749484664353e-06 }, { "step": 400, "losses": { "prediction_mse": 0.009409072808921337, "latent_mse": 0.11286382377147675, "scaled_prediction": 0.0013626933796331286, "scaled_weighted_latent": 0.0016345795011147857, "total": 0.0029972728807479143 }, "gradients": { "norms": { "prediction": 0.0029125979635864496, "weighted_latent": 0.002912567462772131, "total": 5.532348268388887e-07 }, "prediction_weighted_latent_dot": -8.483139026793651e-06, "prediction_weighted_latent_cosine": -1.0, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.552038338559214e-05 }, { "step": 500, "losses": { "prediction_mse": 0.009409597143530846, "latent_mse": 0.11285925656557083, "scaled_prediction": 0.0013627692824229598, "scaled_weighted_latent": 0.001634513377211988, "total": 0.0029972826596349478 }, "gradients": { "norms": { "prediction": 0.0029151239432394505, "weighted_latent": 0.00291238259524107, "total": 3.6262848880141973e-05 }, "prediction_weighted_latent_dot": -8.489301762892865e-06, "prediction_weighted_latent_cosine": -0.9999229121623138, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.001324942335486412 }, { "step": 600, "losses": { "prediction_mse": 0.009409203194081783, "latent_mse": 0.11286260187625885, "scaled_prediction": 0.0013627122389152646, "scaled_weighted_latent": 0.001634561805985868, "total": 0.0029972740449011326 }, "gradients": { "norms": { "prediction": 0.002913573756814003, "weighted_latent": 0.0029125947039574385, "total": 1.4871639905322809e-05 }, "prediction_weighted_latent_dot": -8.48594936542213e-06, "prediction_weighted_latent_cosine": -0.9999870224497254, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0002832206664606929 }, { "step": 700, "losses": { "prediction_mse": 0.009409106336534023, "latent_mse": 0.1128636971116066, "scaled_prediction": 0.0013626981526613235, "scaled_weighted_latent": 0.0016345777548849583, "total": 0.002997275907546282 }, "gradients": { "norms": { "prediction": 0.0029136694502085447, "weighted_latent": 0.0029126203153282404, "total": 2.4933979148045182e-05 }, "prediction_weighted_latent_dot": -8.48610307002673e-06, "prediction_weighted_latent_cosine": -0.9999634989708791, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005798392812721431 }, { "step": 800, "losses": { "prediction_mse": 0.009408985264599323, "latent_mse": 0.11286468803882599, "scaled_prediction": 0.0013626806903630495, "scaled_weighted_latent": 0.001634592073969543, "total": 0.0029972726479172707 }, "gradients": { "norms": { "prediction": 0.002912086434662342, "weighted_latent": 0.0029125665314495564, "total": 5.200785381020978e-06 }, "prediction_weighted_latent_dot": -8.481631994072814e-06, "prediction_weighted_latent_cosine": -0.9999984092458225, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0002596588456071913 }, { "step": 900, "losses": { "prediction_mse": 0.0094091035425663, "latent_mse": 0.11286351084709167, "scaled_prediction": 0.001362697803415358, "scaled_weighted_latent": 0.0016345750773325562, "total": 0.0029972728807479143 }, "gradients": { "norms": { "prediction": 0.0029128547757864, "weighted_latent": 0.002912570722401142, "total": 3.2265631944028428e-06 }, "prediction_weighted_latent_dot": -8.483890269417316e-06, "prediction_weighted_latent_cosine": -0.9999993789239304, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 9.733595106808934e-06 }, { "step": 1000, "losses": { "prediction_mse": 0.009409232996404171, "latent_mse": 0.1128622367978096, "scaled_prediction": 0.0013627165462821722, "scaled_weighted_latent": 0.0016345565672963858, "total": 0.002997273113578558 }, "gradients": { "norms": { "prediction": 0.0029134999494999647, "weighted_latent": 0.0029125595465302467, "total": 9.305387720814906e-06 }, "prediction_weighted_latent_dot": -8.485700163873844e-06, "prediction_weighted_latent_cosine": -0.9999950590228625, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00013846109504811466 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "84f3f61efe69cfb8f109725e0b62863a3c197f51886238d947df9d8dc3240346" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "0c60138bb638f11784fcfe838ffdaae05a2da62c6263090b727a58b9d26074ef" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "60cd5c13669bbe0de96ac584b56c31330afd0dd897e43e695677ad3a365f3c17" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "64399e82cd4edecbbd1334b2d2a2d885d878a6eaf5c9d9a973222def9cda2e11" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "a4aedf808d6ebcf878363e26ace828f34c789fe8720ed3da981ca429a440281c" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "05c847ae687a7b59790af4019f6e7795946c32dfe8ba4100826f763410faac6d" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "d2fa4d98d2d69f617fed60578d82d1b26a8e6acc0b8d29dbb1673f0312e11b4e" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "2a38473cfdf6a9bc698e342a3a977cbfd4fadca73de1505d3ed86fd67e75e761" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "9a26bfb6186e188a8ce57423da3ed695b7aaa73c8706db55d7771e6b90ad10ff" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "f72d265aec46e0b626d2109fbfac3943e73d2f4a26ed25030ca6fe4fb87ab0fc" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "ab7b2479826ff73a906f670b8db6e1130c3c6aa0816dbc06b152436568d135a0" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "fe943a3c3a39ef1a0d8246e36a9c3703472e67a373e38cf26ebb5945840de54e" } ], "final_losses": { "prediction_mse": 0.009409232996404171, "latent_mse": 0.1128622367978096, "scaled_prediction": 0.0013627165462821722, "scaled_weighted_latent": 0.0016345565672963858, "total": 0.002997273113578558 }, "training_seconds": 3.096993900020607, "evaluation_seconds": 0.49256959999911487 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 3, "arm": "latent_only", "status": "passed", "files": { "seed-3-latent_only/result.json": "ae9931d953b8170cc438315f9384d126a1a827943bf228553fc8772e9f115419", "seed-3-latent_only/checkpoint.json": "47ee7af881cc62c1204047cb50ec63fbe8bd83e23095233d8a67330c15345ed2", "inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "seed-3-latent_only/diagnostics-arrays.npz": "aceda8f5bc2a00605c3e40b06fb7a9c70d8bce29d8f62ff2cee7356d0a5a4799" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e", "parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c", "parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 5.811452865600586e-07, "max_tolerance_ratio": 0.3990840958602454, "worst_tolerance_index": [ 60, 2 ], "actual_at_worst": 0.004562556743621826, "saved_at_worst": 0.004561975598335266, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 0.00010696637036744505, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 0.00010696651588659734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014397905845926648 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001894764427561313, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 0.00018947666103485972, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-10, "numeric_tolerance_ratio": 0.00021421976386698359 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2893109321594238, "saved_at_worst": 1.2893109321594238, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4014429971381162, "worst_tolerance_index": [ 615, 4 ], "actual_at_worst": 0.0012289471924304962, "saved_at_worst": 0.0012284964323043823, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 0.00010002103954320773, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "reported": { "count": 784, "observable_mse": 0.00010002109047491103, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.042732370857766e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 8.433461334789172e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005999956279993057 }, "reported": { "count": 580, "observable_mse": 8.433467883151025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0059999567456543446 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002910390913614281 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 0.00011633458780124784, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "reported": { "count": 112, "observable_mse": 0.00011633463145699352, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006062434520572424 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3153718491430616e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 0.00017905373533722013, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "reported": { "count": 92, "observable_mse": 0.00017905370623338968, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006442381534725428 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.8591882208963364e-05 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.000102343074104283, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "reported": { "count": 98, "observable_mse": 0.00010234319051960483, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.001757528884955003 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.67251874296926e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897133611142635 }, "reported": { "count": 70, "observable_mse": 8.67253402248025e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0058971368707716465 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.0020504503657808367 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010630184988258407, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897137802094221 }, "reported": { "count": 14, "observable_mse": 0.00010630159522406757, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897134076803923 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002343372258461114 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017647375352680683, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897135008126497 }, "reported": { "count": 14, "observable_mse": 0.00017647400090936571, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005897139199078083 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002636292941319088 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252822539769113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.737066673347726e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 70, "observable_mse": 8.737093594390899e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968606099486351 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0005832209579303454 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010268489859299734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "reported": { "count": 14, "observable_mse": 0.0001026846730383113, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968607030808926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.255546860396862e-10, "numeric_tolerance_ratio": 0.00022326212619665206 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017815936007536948, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "reported": { "count": 14, "observable_mse": 0.00017815965111367404, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005968604236841202 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.0002859439373556026 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001030568455462344, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580707758665 }, "reported": { "count": 98, "observable_mse": 0.00010305678733857349, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.00579952893372632 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.592774247517809e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058580242097378 }, "reported": { "count": 70, "observable_mse": 8.592770609539002e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585904866456985 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.006379481642109011 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001099406072171405, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585797764360905 }, "reported": { "count": 14, "observable_mse": 0.00010994031617883593, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060585867613554 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.004349647583201486 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00018181865743827075, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058583967387676 }, "reported": { "count": 14, "observable_mse": 0.00018181874474976212, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058590020984411 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.003769693806922108 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200838732998818, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097275298088789 }, "reported": { "count": 98, "observable_mse": 0.00010200848191743717, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006097279489040375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.002603515450311345 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543613512301818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060906685888767 }, "reported": { "count": 72, "observable_mse": 8.543617150280625e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006060910876840353 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026094108969952367 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0001222111313836649, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00609727343544364 }, "reported": { "count": 14, "observable_mse": 0.00012221158249303699, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060972776263952255 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0026035157515688335 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00017787207616493106, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311086472123861 }, "reported": { "count": 12, "observable_mse": 0.00017787201795727015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006311097647994757 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006851697621195759 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010200543329119682, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094702053815126 }, "reported": { "count": 98, "observable_mse": 0.00010200551332673058, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094708573073149 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004050559842515715 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.543937292415649e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058296654373407 }, "reported": { "count": 72, "observable_mse": 8.543945295969024e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006058302707970142 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.05359673500061e-09, "numeric_tolerance_ratio": 0.0037697612537818566 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00012222511577419937, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060947067104279995 }, "reported": { "count": 14, "observable_mse": 0.00012222547957208008, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006094712298363447 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.0034719076328323475 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.0001778121804818511, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006308719515800476 }, "reported": { "count": 12, "observable_mse": 0.0001778119767550379, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0063087306916713715 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.006852692037580273 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067623384296894 }, "reported": { "count": 98, "observable_mse": 0.0001024944067467004, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0023185078919724916 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 8.603723108535632e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067622918635607 }, "reported": { "count": 70, "observable_mse": 8.603732567280531e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067619659006596 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.259629011154175e-09, "numeric_tolerance_ratio": 0.00202869440547593 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00010919466149061918, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067620124667883 }, "reported": { "count": 14, "observable_mse": 0.00010919448686763644, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.00028981346969819097 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00017808005213737488, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006067628972232342 }, "reported": { "count": 14, "observable_mse": 0.00017807971744332463, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00606762059032917 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.381903171539307e-09, "numeric_tolerance_ratio": 0.005216642454567437 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875155178131536, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0121191280632047e-06 }, "reported": { "count": 98, "observable_mse": 0.00011875170457642525, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0175972420256585e-06 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.47811396245379e-09, "numeric_tolerance_ratio": 0.005477556567808147 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.00010696427489165217, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0115189752468723e-06 }, "reported": { "count": 84, "observable_mse": 0.00010696444951463491, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0180974641116336e-06 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.5784888647613116e-09, "numeric_tolerance_ratio": 0.006577819178658794 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.00018947529315482825, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0157128826904227e-06 }, "reported": { "count": 14, "observable_mse": 0.00018947523494716734, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.0145910209757858e-06 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.121861714636907e-09, "numeric_tolerance_ratio": 0.0011217479031018784 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 6.69803557684645e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705172687768936 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.005974748847249882 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 4.724697646452114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754040114581585 }, "reported": { "count": 72, "observable_mse": 4.724687460111454e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008754054084420204 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-08, "numeric_tolerance_ratio": 0.007448969996752607 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 6.864315218990669e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00870516151189804 }, "reported": { "count": 14, "observable_mse": 6.864342140033841e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008705178275704384 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6763806343078613e-08, "numeric_tolerance_ratio": 0.008962120593553839 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 0.00018344067211728543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405925706028938 }, "reported": { "count": 12, "observable_mse": 0.00018344077398069203, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.008405914530158043 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1175870895385742e-08, "numeric_tolerance_ratio": 0.0060718911179741195 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869952917099, "saved_at_worst": 1.2869952917099, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.48761554050232203, "worst_tolerance_index": [ 3767, 3 ], "actual_at_worst": -0.00039064884185791016, "saved_at_worst": -0.00039014220237731934, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.00011224200716242194, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010571317747235298 }, "reported": { "count": 6272, "observable_mse": 0.00011224204354221001, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010571316815912724 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004527287110249859 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 9.662971569923684e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01050430815666914 }, "reported": { "count": 4092, "observable_mse": 9.662973752710968e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010504309087991714 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00045420822063246434 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.00011953953799093142, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010571317747235298 }, "reported": { "count": 896, "observable_mse": 0.00011953959619859233, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010571317747235298 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.7520068339208964e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.0001569047017255798, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010782089084386826 }, "reported": { "count": 1284, "observable_mse": 0.00015690474538132548, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.010782088153064251 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00044813714952804585 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2941532135009766, "saved_at_worst": 1.2941532135009766, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.47592563353345485, "worst_tolerance_index": [ 520, 2 ], "actual_at_worst": -0.0003317594528198242, "saved_at_worst": -0.0003322511911392212, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00019525222887750715, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018011802807450294 }, "reported": { "count": 3136, "observable_mse": 0.00019525224342942238, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018011802807450294 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4273227263940613e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.0001820577308535576, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0181255042552948 }, "reported": { "count": 1582, "observable_mse": 0.00018205780361313373, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0181255042552948 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.145861626911538e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00019539719505701214, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018011802807450294 }, "reported": { "count": 448, "observable_mse": 0.00019539731147233397, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.018011802807450294 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011418419338689126 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.00021406661835499108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017847908660769463 }, "reported": { "count": 1106, "observable_mse": 0.00021406660380307585, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.017847908660769463 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4246935909883959e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2845898866653442, "saved_at_worst": 1.2845898866653442, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.40870005807900295, "worst_tolerance_index": [ 1978, 3 ], "actual_at_worst": -0.00093841552734375, "saved_at_worst": -0.0009379684925079346, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0004240799753461033, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028190135955810547 }, "reported": { "count": 3136, "observable_mse": 0.0004240800335537642, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028190135955810547 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.583961436031236e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.0004079900973010808, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02793256938457489 }, "reported": { "count": 1001, "observable_mse": 0.00040799021371640265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02793256938457489 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011185187479664834 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0004217948007863015, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028190141543745995 }, "reported": { "count": 448, "observable_mse": 0.0004217949463054538, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028190141543745995 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.000139629644445514 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.0004342339816503227, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028341861441731453 }, "reported": { "count": 1687, "observable_mse": 0.0004342339525464922, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.028341861441731453 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-11, "numeric_tolerance_ratio": 2.7892637436628367e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.328358769416809, "saved_at_worst": 1.328358769416809, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 6.407499313354492e-07, "max_tolerance_ratio": 0.4102182073710469, "worst_tolerance_index": [ 788, 3 ], "actual_at_worst": 0.001624464988708496, "saved_at_worst": 0.001623988151550293, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0010140761733055115, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04686106741428375 }, "reported": { "count": 3136, "observable_mse": 0.0010140760568901896, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04686106741428375 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00010569685666380313 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0009457325213588774, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04408523067831993 }, "reported": { "count": 407, "observable_mse": 0.0009457324049435556, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04408523067831993 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.0001063568133406557 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0010128346038982272, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04686106741428375 }, "reported": { "count": 448, "observable_mse": 0.0010128346038982272, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04686107113957405 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006551565462630186 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0010265145683661103, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04733925312757492 }, "reported": { "count": 2281, "observable_mse": 0.0010265145683661103, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04733925312757492 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011598924174904823 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010999653022736311, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010999666119460016, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00012954231484367922 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011603816412389278 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010919027408817783, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010919012856902555, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014394738889362153 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010782991535961628 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011893396003870293, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011893427290488034, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003091888621587732 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.010790933854877949 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011383872333681211, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011383901437511668, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.9103830456733704e-10, "numeric_tolerance_ratio": 0.00028776244525315775 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.03806070610880852 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00042644605855457485, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.000426446640631184, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-10, "numeric_tolerance_ratio": 0.0005582693981920547 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0369470976293087 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0004731984226964414, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00047319853911176324, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011115546162157263 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.035910893231630325 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0003548956592567265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00035489487345330417, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.8580342233181e-10, "numeric_tolerance_ratio": 0.0007588714631438345 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020458154380321503 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00018904275202658027, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00018904254829976708, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.00019994696482166673 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 2.021566388066276e-06 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011874968186020851, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011874980555148795, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2369127944111824e-10, "numeric_tolerance_ratio": 0.00012223968555211934 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006445486564189196 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00012876497930847108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00012876487744506449, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.00010056843833486492 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011875350901391357, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011875366908498108, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6007106751203537e-10, "numeric_tolerance_ratio": 0.0001581924738429869 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.008087845519185066 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 9.686173143563792e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 9.68617750913836e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.3655745685100555e-11, "numeric_tolerance_ratio": 4.3236944961253016e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 6.698037759633735e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.698032666463405e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.059283086545828e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012188127264380455 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00014413957251235843, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00014413951430469751, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.738058002000685e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010252827632939443, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010252842912450433, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.5279510989785194e-10, "numeric_tolerance_ratio": 0.00015124442457133805 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00011474297207314521, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00011474302300484851, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.035392711752096e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010750066576292738, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00010750071669463068, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.0390007111411554e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00010249445040244609, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001024944576784037, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2021396741805025e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.00014975426893215626, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0001497541816206649, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.602325712311944e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 0.00010700838174670935, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 0.00010700869461288676, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.128661774098873e-10, "numeric_tolerance_ratio": 0.0003095536838477713 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 6.998753087827936e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 6.998747994657606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.057772256490849e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.253204239532351e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.253222429426387e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-10, "numeric_tolerance_ratio": 0.0001805890875343717 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0002318347105756402, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.00023183472512755543, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.422219535332207e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.011943741701543331, "latent_mse": 0.212107315659523, "scaled_prediction": 0.0017297833692282438, "scaled_weighted_latent": 0.0030718990601599216, "total": 0.0030718990601599216 }, "gradients": { "norms": { "prediction": 0.0034363465383648872, "weighted_latent": 0.004243341740220785, "total": 0.004243341740220785 }, "prediction_weighted_latent_dot": -1.2367510862532072e-05, "prediction_weighted_latent_cosine": -0.8481591220444237, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.012406026013195515, "latent_mse": 0.20755654573440552, "scaled_prediction": 0.0017967348685488105, "scaled_weighted_latent": 0.003005991457030177, "total": 0.003005991457030177 }, "gradients": { "norms": { "prediction": 0.004003142472356558, "weighted_latent": 0.0041850716806948185, "total": 0.0041850716806948185 }, "prediction_weighted_latent_dot": -1.3552008567785379e-05, "prediction_weighted_latent_cosine": -0.8089090973549798, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.031445253640413284 }, { "step": 100, "losses": { "prediction_mse": 0.027742937207221985, "latent_mse": 0.04495343565940857, "scaled_prediction": 0.00401794258505106, "scaled_weighted_latent": 0.0006510497769340873, "total": 0.0006510497769340873 }, "gradients": { "norms": { "prediction": 0.011764733120799065, "weighted_latent": 0.0018284975085407495, "total": 0.0018284975085407495 }, "prediction_weighted_latent_dot": 1.0727669177867938e-05, "prediction_weighted_latent_cosine": 0.4986880018611385, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.005691013298928738 }, { "step": 200, "losses": { "prediction_mse": 0.007840918377041817, "latent_mse": 0.008958370424807072, "scaled_prediction": 0.0011355812894180417, "scaled_weighted_latent": 0.0001297419221373275, "total": 0.0001297419221373275 }, "gradients": { "norms": { "prediction": 0.008955360390245914, "weighted_latent": 0.000795930449385196, "total": 0.000795930449385196 }, "prediction_weighted_latent_dot": 4.943171006743796e-06, "prediction_weighted_latent_cosine": 0.6935015683567417, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0028745271265506744 }, { "step": 300, "losses": { "prediction_mse": 0.001478584366850555, "latent_mse": 0.0014538821997120976, "scaled_prediction": 0.00021413981448858976, "scaled_weighted_latent": 2.105622479575686e-05, "total": 2.105622479575686e-05 }, "gradients": { "norms": { "prediction": 0.00454098591580987, "weighted_latent": 0.0003189259150531143, "total": 0.0003189259150531143 }, "prediction_weighted_latent_dot": 9.54590291257773e-07, "prediction_weighted_latent_cosine": 0.659139059299341, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0013068236876279116 }, { "step": 400, "losses": { "prediction_mse": 0.0002792561717797071, "latent_mse": 0.0001837616873672232, "scaled_prediction": 4.044399975100532e-05, "scaled_weighted_latent": 2.6613761292537674e-06, "total": 2.6613761292537674e-06 }, "gradients": { "norms": { "prediction": 0.0016344879986718297, "weighted_latent": 0.00011334959708619863, "total": 0.00011334959708619863 }, "prediction_weighted_latent_dot": 1.2311144814702857e-07, "prediction_weighted_latent_cosine": 0.6645026589731049, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005251585971564054 }, { "step": 500, "losses": { "prediction_mse": 0.0001206986780744046, "latent_mse": 1.7532936908537522e-05, "scaled_prediction": 1.7480499082012102e-05, "scaled_weighted_latent": 2.5392529323653434e-07, "total": 2.5392529323653434e-07 }, "gradients": { "norms": { "prediction": 0.0005504697328433394, "weighted_latent": 3.501208630041219e-05, "total": 3.501208630041219e-05 }, "prediction_weighted_latent_dot": 1.0782380321927576e-08, "prediction_weighted_latent_cosine": 0.5594524905161186, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0001831014669733122 }, { "step": 600, "losses": { "prediction_mse": 0.0001069319769158028, "latent_mse": 1.23396443996171e-06, "scaled_prediction": 1.548670115880668e-05, "scaled_weighted_latent": 1.787121028939964e-08, "total": 1.787121028939964e-08 }, "gradients": { "norms": { "prediction": 0.0003244292747694999, "weighted_latent": 9.28842473513214e-06, "total": 9.28842473513214e-06 }, "prediction_weighted_latent_dot": 4.0361963771218257e-10, "prediction_weighted_latent_cosine": 0.13393996656636023, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.463738852995448e-05 }, { "step": 700, "losses": { "prediction_mse": 0.0001066865079337731, "latent_mse": 6.252758311120488e-08, "scaled_prediction": 1.5451149010914378e-05, "scaled_weighted_latent": 9.055718952488689e-10, "total": 9.055718952488689e-10 }, "gradients": { "norms": { "prediction": 0.00031490897526964545, "weighted_latent": 2.0908673832309432e-06, "total": 2.0908673832309432e-06 }, "prediction_weighted_latent_dot": -7.719813677198317e-11, "prediction_weighted_latent_cosine": -0.11724525942827876, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.3811811186315026e-05 }, { "step": 800, "losses": { "prediction_mse": 0.00010689532064134255, "latent_mse": 2.2136834587627163e-09, "scaled_prediction": 1.548139152873773e-05, "scaled_weighted_latent": 3.206024334900803e-11, "total": 3.206024334900803e-11 }, "gradients": { "norms": { "prediction": 0.00031767296604812145, "weighted_latent": 3.934148367079615e-07, "total": 3.934148367079615e-07 }, "prediction_weighted_latent_dot": -2.4161134898337266e-11, "prediction_weighted_latent_cosine": -0.19332425172822873, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.892859583880636e-06 }, { "step": 900, "losses": { "prediction_mse": 0.00010695397213567048, "latent_mse": 5.3057332832784354e-11, "scaled_prediction": 1.548988620925229e-05, "scaled_weighted_latent": 7.684165803656384e-13, "total": 7.684165803656384e-13 }, "gradients": { "norms": { "prediction": 0.00031847888021729887, "weighted_latent": 6.091948279163262e-08, "total": 6.091948279163262e-08 }, "prediction_weighted_latent_dot": -4.365838853631621e-12, "prediction_weighted_latent_cosine": -0.22502504459760064, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 4.838051381739206e-07 }, { "step": 1000, "losses": { "prediction_mse": 0.00010696444951463491, "latent_mse": 1.0365225388067612e-12, "scaled_prediction": 1.5491403246414848e-05, "scaled_weighted_latent": 1.5011705732106105e-14, "total": 1.5011705732106105e-14 }, "gradients": { "norms": { "prediction": 0.00031862015021033585, "weighted_latent": 8.613119106826161e-09, "total": 8.613119106826161e-09 }, "prediction_weighted_latent_dot": -7.386099110801014e-13, "prediction_weighted_latent_cosine": -0.2691419781087464, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.789324812762061e-07 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "2609ad2704a6063d91847fcb51c52e57f5c8517e276eeac245d2e327edc58ff3" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "0e3d2dd4cd57267cdee0643d8eb23e0eedae1c115d6523a89036f6aac1a3c9f7" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "190491187f8fa1bc836b3998b5b65b4c4d8d66ee4002184fe47efdee70e0673d" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "87b1abc43b8d15560398941b9ef832fad3edc7904f08de0aedeb6e5c40d30b58" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "c6895352e70635b188d04d580c8d801cc9a80f9f364141413686848987776b5e" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "8489835f222439a3081472475cd77a99349a411fa927df6e32da8ef302dbaf22" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "3cdec08a90488cf6c84fd797c4c3426ac47fc64150ef8179e962c6f21edcf853" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "91250f181ff3f14daf95381550c85612b8d4984e7ff0d71f69da0890e3584655" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "2681ea448321713221cb5be69c7df1d8569752ebbbd5b9e197f3408316bd75cc" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "515e6f3c6c67a5a1e2deec322265e5843cf03a3f6982bd682f6af5a09a1801fd" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "ccc418c8dbd904af9ff6babf08f282bb312f490a7bb966d09f2aef1bbc724571" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "47ee7af881cc62c1204047cb50ec63fbe8bd83e23095233d8a67330c15345ed2" } ], "final_losses": { "prediction_mse": 0.00010696444951463491, "latent_mse": 1.0365225388067612e-12, "scaled_prediction": 1.5491403246414848e-05, "scaled_weighted_latent": 1.5011705732106105e-14, "total": 1.5011705732106105e-14 }, "training_seconds": 2.7897566000465304, "evaluation_seconds": 0.4940434000454843 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 4, "arm": "native", "status": "passed", "files": { "seed-4-native/result.json": "9bbda5214373b1ace29e9b95c30b3eb6e115f9f4fd512100b05bb6468ef36a92", "seed-4-native/checkpoint.json": "2b443c6141bcba042a8d14a0d7e0d065b831ee6ad11c793c91e2bf196bf229ad", "inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "seed-4-native/diagnostics-arrays.npz": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.0978193283081055e-07, "max_tolerance_ratio": 0.3144731636939299, "worst_tolerance_index": [ 0, 8 ], "actual_at_worst": 0.00018735229969024658, "saved_at_worst": 0.0001876726746559143, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450094886356965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.3450114895240404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.996207215045689e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800791404908523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 5.800786675536074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.702096589973028e-05 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2923264503479004, "saved_at_worst": 1.2923264503479004, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.37291832992681323, "worst_tolerance_index": [ 343, 9 ], "actual_at_worst": 0.001188725233078003, "saved_at_worst": 0.0011883080005645752, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 8.654144039610401e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043472398072481155 }, "reported": { "count": 784, "observable_mse": 8.654144039610401e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043472398072481155 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 9.035727271111682e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04677638038992882 }, "reported": { "count": 580, "observable_mse": 9.035727271111682e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.046776384115219116 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006561337704954932 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 9.340440010419115e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04347239434719086 }, "reported": { "count": 112, "observable_mse": 9.340439282823354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043472398072481155 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006966753750995664 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 5.4130203352542594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003164678579196334 }, "reported": { "count": 92, "observable_mse": 5.413021062850021e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031646783463656902 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017686010818346055 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5274550353060476e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211733050644398 }, "reported": { "count": 98, "observable_mse": 2.5274570361943915e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171675249934 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012327307636645429 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0186165784252807e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "reported": { "count": 70, "observable_mse": 2.0186189431115054e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211709767580032 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002817670465808772 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1840363842784427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171209588647 }, "reported": { "count": 14, "observable_mse": 2.1840449335286394e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221172373741865 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0008805219275269677 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.415066334535368e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221169812604785 }, "reported": { "count": 14, "observable_mse": 5.415061968960799e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.003698191444345532 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493371140328236e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220667831599712 }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.008045871341972934 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9936096578021534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022065619006753 }, "reported": { "count": 70, "observable_mse": 1.9936062017222866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220558401197195 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.778887033462524e-09, "numeric_tolerance_ratio": 0.007509480187709456 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.3793707441654988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022068180143833 }, "reported": { "count": 14, "observable_mse": 2.3793658328941092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009118654187569325 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.497939855558798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022070275619626 }, "reported": { "count": 14, "observable_mse": 5.497950769495219e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220600310713053 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.007867071950464247 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5550732971169055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836770940572023 }, "reported": { "count": 98, "observable_mse": 2.55507566180313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836845446377993 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.005651364435865696 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.939376124937553e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836775597184896 }, "reported": { "count": 70, "observable_mse": 1.9393763068364933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836838461458683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004768338995395631 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.679098724911455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836782582104206 }, "reported": { "count": 14, "observable_mse": 2.679110002645757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836875714361668 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.007064203922984993 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.509533730219118e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836743000894785 }, "reported": { "count": 14, "observable_mse": 5.509537731995806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003183686174452305 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009006860956200402 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5143033781205304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363468684256077 }, "reported": { "count": 98, "observable_mse": 2.514301559131127e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634756691753864 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.005226873219612861 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949908801179845e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588928636163473 }, "reported": { "count": 72, "observable_mse": 1.949909346876666e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588993828743696 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004880086177358148 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.875245627365075e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634670544415712 }, "reported": { "count": 14, "observable_mse": 2.8752372600138187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363472642377019 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004181499522790897 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.4795717005617917e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033907967153936625 }, "reported": { "count": 12, "observable_mse": 5.4795644246041775e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003390809055417776 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.00921529390986458 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5130846552201547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 98, "observable_mse": 2.5130850190180354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750871662050486 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0031334013256818006 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949693978531286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003370406571775675 }, "reported": { "count": 72, "observable_mse": 1.949694888025988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033704107627272606 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.003134497256772981 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750897273421288 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.752088665962219e-09, "numeric_tolerance_ratio": 0.005048256724707581 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.473628334584646e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003403007285669446 }, "reported": { "count": 12, "observable_mse": 5.4736312449676916e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034030096139758825 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0017371519558643553 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509041734039783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103164955973625 }, "reported": { "count": 98, "observable_mse": 2.5509052647976205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103166352957487 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0010661422012763408 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9929024347220547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031647231429815 }, "reported": { "count": 70, "observable_mse": 1.9929013433284126e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031661201268435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.001066142220220661 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.406693783996161e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103160299360752 }, "reported": { "count": 14, "observable_mse": 2.406714338576421e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031658872962 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004264568956659927 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.485124347615056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103169146925211 }, "reported": { "count": 14, "observable_mse": 5.485117799253203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031689140945673 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001776903321481438 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0005257376469671726, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265803664922714 }, "reported": { "count": 98, "observable_mse": 0.0005257370648905635, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265802174806595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0011232763007838916 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 0.0005169388605281711, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265803664922714 }, "reported": { "count": 84, "observable_mse": 0.0005169383366592228, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265802919864655 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0005616381188482063 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 0.0005785307730548084, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265803664922714 }, "reported": { "count": 14, "observable_mse": 0.0005785297835245728, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.12265802174806595 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0011232763007838916 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492266927845776e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384734462946653 }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006583761406554621 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.54845256981207e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003532622940838337 }, "reported": { "count": 72, "observable_mse": 8.548483492631931e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035326124634593725 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.0077423180429610685 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4730835573573131e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003438472282141447 }, "reported": { "count": 14, "observable_mse": 1.4730812836205587e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.683411240577698e-09, "numeric_tolerance_ratio": 0.005717477010955329 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.987682041246444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080525808036327 }, "reported": { "count": 12, "observable_mse": 4.987671127310023e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080549091100693 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0018178454519917982 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2935075759887695, "saved_at_worst": 1.2935075759887695, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.5794924900882756, "worst_tolerance_index": [ 5875, 12 ], "actual_at_worst": 0.0005421936511993408, "saved_at_worst": 0.0005428045988082886, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 0.0003078756562899798, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0645698681473732 }, "reported": { "count": 6272, "observable_mse": 0.00030787562718614936, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0645698606967926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0009991410104973272 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 0.00035424259840510786, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.06916395574808121 }, "reported": { "count": 4092, "observable_mse": 0.00035424259840510786, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0691639631986618 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0009411581098114823 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 0.00031304298317991197, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0645698606967926 }, "reported": { "count": 896, "observable_mse": 0.00031304298317991197, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0645698606967926 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 0.00015650216664653271, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04702414572238922 }, "reported": { "count": 1284, "observable_mse": 0.0001565021084388718, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.04702414199709892 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006532830075113513 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.1930153369903564, "saved_at_worst": 1.1930153369903564, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4165565519257677, "worst_tolerance_index": [ 1728, 8 ], "actual_at_worst": 1.5810132026672363e-05, "saved_at_worst": 1.6227364540100098e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0007142377435229719, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08957353979349136 }, "reported": { "count": 3136, "observable_mse": 0.00071423762710765, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08957353234291077 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.0007482491000987653 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 0.0007724000606685877, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09334152936935425 }, "reported": { "count": 1582, "observable_mse": 0.0007724001188762486, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.09334152936935425 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.4034068797232435e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0007104201940819621, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08957353979349136 }, "reported": { "count": 448, "observable_mse": 0.0007104201940819621, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08957353979349136 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 0.0006325896247290075, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08389020711183548 }, "reported": { "count": 1106, "observable_mse": 0.0006325897411443293, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.08389020711183548 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.0001094891504902602 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.3042800426483154, "saved_at_worst": 1.3042800426483154, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.409652597231079, "worst_tolerance_index": [ 1727, 0 ], "actual_at_worst": -0.0006401576101779938, "saved_at_worst": -0.0006397217512130737, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.005111921578645706, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.18345704674720764 }, "reported": { "count": 3136, "observable_mse": 0.005111921112984419, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.18345703184604645 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0007702568912411534 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00642401585355401, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.2068207710981369 }, "reported": { "count": 1001, "observable_mse": 0.00642401585355401, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.2068207710981369 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.005084032658487558, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.18345704674720764 }, "reported": { "count": 448, "observable_mse": 0.005084032658487558, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.18345703184604645 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4901161193847656e-08, "numeric_tolerance_ratio": 0.0007702568912411534 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.004340781830251217, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16806550323963165 }, "reported": { "count": 1687, "observable_mse": 0.004340781830251217, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.16806550323963165 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2052175998687744, "saved_at_worst": 1.2052175998687744, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.662441253662109e-07, "max_tolerance_ratio": 0.42896605801972054, "worst_tolerance_index": [ 3103, 0 ], "actual_at_worst": -0.0004641897976398468, "saved_at_worst": -0.00046463869512081146, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.014956056140363216, "exact_state_accuracy": 0.9764030612244898, "per_field_accuracy": { "x": 1.0, "y": 0.9920280612244898, "h": 0.9808673469387755 }, "normalized_latent_rmse": 0.2683657109737396 }, "reported": { "count": 3136, "observable_mse": 0.014956055209040642, "exact_state_accuracy": 0.9764030575752258, "per_field_accuracy": { "x": 1.0, "y": 0.9920280575752258, "h": 0.980867326259613 }, "normalized_latent_rmse": 0.2683657109737396 }, "differences": [], "max_errors": { "accuracy_abs": 2.0679162493486558e-08, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00037318501133868916 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.020802950486540794, "exact_state_accuracy": 0.9533169533169533, "per_field_accuracy": { "x": 1.0, "y": 0.9778869778869779, "h": 0.9680589680589681 }, "normalized_latent_rmse": 0.3151726722717285 }, "reported": { "count": 407, "observable_mse": 0.020802950486540794, "exact_state_accuracy": 0.9533169269561768, "per_field_accuracy": { "x": 1.0, "y": 0.977886974811554, "h": 0.9680589437484741 }, "normalized_latent_rmse": 0.3151726722717285 }, "differences": [], "max_errors": { "accuracy_abs": 2.6360776561595856e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.015006010420620441, "exact_state_accuracy": 0.9776785714285714, "per_field_accuracy": { "x": 1.0, "y": 0.9933035714285714, "h": 0.9821428571428571 }, "normalized_latent_rmse": 0.2683657109737396 }, "reported": { "count": 448, "observable_mse": 0.015006013214588165, "exact_state_accuracy": 0.9776785969734192, "per_field_accuracy": { "x": 1.0, "y": 0.9933035969734192, "h": 0.9821429252624512 }, "normalized_latent_rmse": 0.2683657109737396 }, "differences": [], "max_errors": { "accuracy_abs": 6.811959407659884e-08, "numeric_abs": 2.7939677238464355e-09, "numeric_tolerance_ratio": 0.0011173183425402945 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.013902980834245682, "exact_state_accuracy": 0.9802718106093818, "per_field_accuracy": { "x": 1.0, "y": 0.9943007452871547, "h": 0.9829022358614643 }, "normalized_latent_rmse": 0.25912654399871826 }, "reported": { "count": 2281, "observable_mse": 0.013902980834245682, "exact_state_accuracy": 0.9802718162536621, "per_field_accuracy": { "x": 1.0, "y": 0.9943007230758667, "h": 0.9829022288322449 }, "normalized_latent_rmse": 0.25912654399871826 }, "differences": [], "max_errors": { "accuracy_abs": 2.2211288031037668e-08, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006257921922951937 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5511306375847198e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5511330022709444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3586689464189724e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006255257874727249 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5392424504389055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5392444513272494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9958204678516046e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006481279619038105 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5999313947977498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5999332137871534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8142724164315183e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006476297043263912 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.6249348593410105e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.624933586048428e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2699590243860832e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.021607279777526855 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.766223668819293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.7661858338397e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.7834979593753815e-10, "numeric_tolerance_ratio": 0.00037580701583076647 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.019717823714017868 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.143248901935294e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.143239443190396e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.40099235142008e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020352382212877274 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.072931475704536e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.072944572428241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00013004741886984787 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012590174563229084 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.74880874005612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.748832386918366e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-10, "numeric_tolerance_ratio": 0.000235585452088286 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.2127009630203247 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.006515105254948139, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.006515105254948139, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.21456055343151093 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.008191288448870182, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.008191288448870182, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0037216152995824814 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5795047147548757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.579504350956995e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6322416655772068e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492263289866969e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.185452315956354e-12, "numeric_tolerance_ratio": 8.17360690293936e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006914579775184393 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1585754186380655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.158584877382964e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.428962719383602e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493387511232868e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.447291564678985e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5946865207515657e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.594685065560043e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4514255306884354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5181043383781798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5181052478728816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.072102508792652e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0015580033650621772, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0015580025501549244, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.149072527885437e-10, "numeric_tolerance_ratio": 0.0007050588968572434 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 0.0005676934379152954, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 0.0005676930886693299, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.4924596548080444e-10, "numeric_tolerance_ratio": 0.00033048458405294303 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.8929631298524328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.8929516702191904e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1459633242338896e-10, "numeric_tolerance_ratio": 0.00011426576708166119 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.3614365343528334e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.3614378076454159e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2715614306756342e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4222289792087395e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4222278878150973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0898436361148734e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.5869325074600056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.586930688470602e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8124881342346815e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 91, "error_count": 7, "per_field_correct_counts": { "x": 98, "y": 91, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 78, "error_count": 20, "per_field_correct_counts": { "x": 98, "y": 87, "h": 85 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 51, "error_count": 47, "per_field_correct_counts": { "x": 98, "y": 91, "h": 51 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3062, "accuracy": 0.9764030612244898 }, "numpy": { "cases": 3136, "correct": 3062, "accuracy": 0.9764030612244898 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0005169383366592228, "latent_mse": 0.015044992789626122, "scaled_prediction": 7.486693357350305e-05, "scaled_weighted_latent": 0.00021789300080854446, "total": 0.0002927599416580051 }, "gradients": { "norms": { "prediction": 0.001070561120286584, "weighted_latent": 0.0012635241728276014, "total": 0.002058987971395254 }, "prediction_weighted_latent_dot": 7.48418131024664e-07, "prediction_weighted_latent_cosine": 0.5532854864566016, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "2b443c6141bcba042a8d14a0d7e0d065b831ee6ad11c793c91e2bf196bf229ad" } ], "final_losses": { "prediction_mse": 0.0005169383366592228, "latent_mse": 0.015044992789626122, "scaled_prediction": 7.486693357350305e-05, "scaled_weighted_latent": 0.00021789300080854446, "total": 0.0002927599416580051 }, "training_seconds": 4.1199964471161366e-05, "evaluation_seconds": 0.5248835000093095 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 4, "arm": "prediction_latent", "status": "passed", "files": { "seed-4-prediction_latent/result.json": "0afedbfab8403dc148a8ab961ba49342d0bdd11d2011d96f92da68f55931d6b8", "seed-4-prediction_latent/checkpoint.json": "a0d56a6c0cb366f7ef9baa918d85d54904469a947720c2e1064b3d1c99352424", "inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "seed-4-prediction_latent/diagnostics-arrays.npz": "ab86bd115de46443231d8b433015c3dd00a97ac893cfcfcc5e8dede70da8b70d" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.0978193283081055e-07, "max_tolerance_ratio": 0.3144731636939299, "worst_tolerance_index": [ 0, 8 ], "actual_at_worst": 0.00018735229969024658, "saved_at_worst": 0.0001876726746559143, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450094886356965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.3450114895240404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.996207215045689e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800791404908523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 5.800786675536074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.702096589973028e-05 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2923264503479004, "saved_at_worst": 1.2923264503479004, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.37291832992681323, "worst_tolerance_index": [ 343, 9 ], "actual_at_worst": 0.001188725233078003, "saved_at_worst": 0.0011883080005645752, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.3999598852242343e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003208009758964181 }, "reported": { "count": 784, "observable_mse": 2.3999595214263536e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003208009758964181 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6292687090974994e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.8472606825525872e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00321482983417809 }, "reported": { "count": 580, "observable_mse": 1.8472603187547065e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00321482983417809 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.631270904443287e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.7871383281308226e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032080088276416063 }, "reported": { "count": 112, "observable_mse": 2.7871372367371805e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032080088276416063 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.088360232795705e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 5.4130203352542594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003164678579196334 }, "reported": { "count": 92, "observable_mse": 5.413021062850021e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031646783463656902 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017686010818346055 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5274550353060476e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211733050644398 }, "reported": { "count": 98, "observable_mse": 2.5274570361943915e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171675249934 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012327307636645429 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0186165784252807e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "reported": { "count": 70, "observable_mse": 2.0186189431115054e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211709767580032 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002817670465808772 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1840363842784427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171209588647 }, "reported": { "count": 14, "observable_mse": 2.1840449335286394e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221172373741865 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0008805219275269677 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.415066334535368e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221169812604785 }, "reported": { "count": 14, "observable_mse": 5.415061968960799e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.003698191444345532 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493371140328236e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220667831599712 }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.008045871341972934 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9936096578021534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022065619006753 }, "reported": { "count": 70, "observable_mse": 1.9936062017222866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220558401197195 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.778887033462524e-09, "numeric_tolerance_ratio": 0.007509480187709456 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.3793707441654988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022068180143833 }, "reported": { "count": 14, "observable_mse": 2.3793658328941092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009118654187569325 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.497939855558798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022070275619626 }, "reported": { "count": 14, "observable_mse": 5.497950769495219e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220600310713053 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.007867071950464247 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5550732971169055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836770940572023 }, "reported": { "count": 98, "observable_mse": 2.55507566180313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836845446377993 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.005651364435865696 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.939376124937553e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836775597184896 }, "reported": { "count": 70, "observable_mse": 1.9393763068364933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836838461458683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004768338995395631 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.679098724911455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836782582104206 }, "reported": { "count": 14, "observable_mse": 2.679110002645757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836875714361668 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.007064203922984993 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.509533730219118e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836743000894785 }, "reported": { "count": 14, "observable_mse": 5.509537731995806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003183686174452305 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009006860956200402 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5143033781205304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363468684256077 }, "reported": { "count": 98, "observable_mse": 2.514301559131127e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634756691753864 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.005226873219612861 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949908801179845e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588928636163473 }, "reported": { "count": 72, "observable_mse": 1.949909346876666e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588993828743696 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004880086177358148 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.875245627365075e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634670544415712 }, "reported": { "count": 14, "observable_mse": 2.8752372600138187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363472642377019 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004181499522790897 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.4795717005617917e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033907967153936625 }, "reported": { "count": 12, "observable_mse": 5.4795644246041775e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003390809055417776 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.00921529390986458 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5130846552201547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 98, "observable_mse": 2.5130850190180354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750871662050486 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0031334013256818006 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949693978531286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003370406571775675 }, "reported": { "count": 72, "observable_mse": 1.949694888025988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033704107627272606 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.003134497256772981 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750897273421288 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.752088665962219e-09, "numeric_tolerance_ratio": 0.005048256724707581 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.473628334584646e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003403007285669446 }, "reported": { "count": 12, "observable_mse": 5.4736312449676916e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034030096139758825 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0017371519558643553 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509041734039783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103164955973625 }, "reported": { "count": 98, "observable_mse": 2.5509052647976205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103166352957487 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0010661422012763408 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9929024347220547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031647231429815 }, "reported": { "count": 70, "observable_mse": 1.9929013433284126e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031661201268435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.001066142220220661 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.406693783996161e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103160299360752 }, "reported": { "count": 14, "observable_mse": 2.406714338576421e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031658872962 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004264568956659927 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.485124347615056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103169146925211 }, "reported": { "count": 14, "observable_mse": 5.485117799253203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031689140945673 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001776903321481438 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5402956453035586e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0029204681050032377 }, "reported": { "count": 98, "observable_mse": 2.5402905521332286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0029204622842371464 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-09, "numeric_tolerance_ratio": 0.004505075718883546 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.0592340661096387e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0029204683378338814 }, "reported": { "count": 84, "observable_mse": 2.059229154838249e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0029204629827290773 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.3551048040390015e-09, "numeric_tolerance_ratio": 0.004144669437308267 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 5.426665302366018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.002920466475188732 }, "reported": { "count": 14, "observable_mse": 5.4266613005893305e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0029204594902694225 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.005406092031693245 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492266927845776e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384734462946653 }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006583761406554621 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.54845256981207e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003532622940838337 }, "reported": { "count": 72, "observable_mse": 8.548483492631931e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035326124634593725 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.0077423180429610685 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4730835573573131e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003438472282141447 }, "reported": { "count": 14, "observable_mse": 1.4730812836205587e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.683411240577698e-09, "numeric_tolerance_ratio": 0.005717477010955329 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.987682041246444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080525808036327 }, "reported": { "count": 12, "observable_mse": 4.987671127310023e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080549091100693 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0018178454519917982 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2935075759887695, "saved_at_worst": 1.2935075759887695, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.5794924900882756, "worst_tolerance_index": [ 5875, 12 ], "actual_at_worst": 0.0005421936511993408, "saved_at_worst": 0.0005428045988082886, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.4210245101130567e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006091923452913761 }, "reported": { "count": 6272, "observable_mse": 2.421025237708818e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006091923452913761 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2583848812010335e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.883342702058144e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006118892692029476 }, "reported": { "count": 4092, "observable_mse": 1.8833430658560246e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.006118892692029476 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6311401245170528e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.712010063987691e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060919225215911865 }, "reported": { "count": 896, "observable_mse": 2.7120109734823927e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060919225215911865 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.070348134056416e-06 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.9315156755037606e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0060051665641367435 }, "reported": { "count": 1284, "observable_mse": 3.9315171306952834e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.006005167029798031 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002909443471853698 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869195938110352, "saved_at_worst": 1.2869195938110352, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4165565519257677, "worst_tolerance_index": [ 1728, 8 ], "actual_at_worst": 1.5810132026672363e-05, "saved_at_worst": 1.6227364540100098e-05, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.7332403735490516e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011979796923696995 }, "reported": { "count": 3136, "observable_mse": 3.7332429201342165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011979796923696995 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5371135039380457e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 3.2660438591847196e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012036066502332687 }, "reported": { "count": 1582, "observable_mse": 3.266046041972004e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.012036066502332687 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.1756814086018716e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.7948124372633174e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011979799717664719 }, "reported": { "count": 448, "observable_mse": 3.7948160752421245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011979798786342144 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.00042371751610127835 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 4.376568176667206e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011898848228156567 }, "reported": { "count": 1106, "observable_mse": 4.37657035945449e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.011898848228156567 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.1827872842550278e-11, "numeric_tolerance_ratio": 2.173275789850249e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2849032878875732, "saved_at_worst": 1.2849032878875732, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4184826686608572, "worst_tolerance_index": [ 1883, 7 ], "actual_at_worst": -0.0005046799778938293, "saved_at_worst": -0.0005042403936386108, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0001141131142503582, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02400389313697815 }, "reported": { "count": 3136, "observable_mse": 0.00011411319428589195, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02400389313697815 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.913252721082345e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 0.00010936548642348498, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02393919788300991 }, "reported": { "count": 1001, "observable_mse": 0.00010936558828689158, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02393919788300991 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0186340659856796e-10, "numeric_tolerance_ratio": 0.00010076142336428203 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0001147212678915821, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02400389313697815 }, "reported": { "count": 448, "observable_mse": 0.00011472133337520063, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02400389313697815 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.548361852765083e-11, "numeric_tolerance_ratio": 6.474090226448136e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 0.00011676864960463718, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.024042200297117233 }, "reported": { "count": 1687, "observable_mse": 0.00011676873691612855, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.024042200297117233 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.731149137020111e-11, "numeric_tolerance_ratio": 8.630373357414125e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2667545080184937, "saved_at_worst": 1.2667545080184937, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.42896605801972054, "worst_tolerance_index": [ 3103, 0 ], "actual_at_worst": -0.0004641897976398468, "saved_at_worst": -0.00046463869512081146, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.00047022575745359063, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04926556348800659 }, "reported": { "count": 3136, "observable_mse": 0.00047022593207657337, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04926556348800659 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016678052973568357 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0004428043612278998, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.048294827342033386 }, "reported": { "count": 407, "observable_mse": 0.0004428044776432216, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.048294827342033386 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011147898256274539 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.00047217815881595016, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04926556348800659 }, "reported": { "count": 448, "observable_mse": 0.0004721784498542547, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04926555976271629 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006285759070490511 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.0004747350758407265, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.049436770379543304 }, "reported": { "count": 2281, "observable_mse": 0.0004747352795675397, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.049436770379543304 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.0372681319713593e-10, "numeric_tolerance_ratio": 0.00019449351965441462 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006257921922951937 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5511306375847198e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5511330022709444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3586689464189724e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006255257874727249 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5392424504389055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5392444513272494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9958204678516046e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006481279619038105 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5999313947977498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5999332137871534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8142724164315183e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006476297043263912 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.6249348593410105e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.624933586048428e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2699590243860832e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.021607279777526855 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.766223668819293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.7661858338397e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.7834979593753815e-10, "numeric_tolerance_ratio": 0.00037580701583076647 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.019717823714017868 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.143248901935294e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.143239443190396e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.40099235142008e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020352382212877274 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.072931475704536e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.072944572428241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00013004741886984787 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012590174563229084 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.74880874005612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.748832386918366e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-10, "numeric_tolerance_ratio": 0.000235585452088286 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.005629351828247309 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7246298486716114e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7246340323472396e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.172307616829235e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.011353629641234875 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 4.365778295323253e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 4.3657921196427196e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.382431946694851e-10, "numeric_tolerance_ratio": 0.0001376422771007888 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0037216152995824814 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5795047147548757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.579504350956995e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6322416655772068e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492263289866969e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.185452315956354e-12, "numeric_tolerance_ratio": 8.17360690293936e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006914579775184393 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1585754186380655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.158584877382964e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.428962719383602e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493387511232868e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.447291564678985e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5946865207515657e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.594685065560043e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4514255306884354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5181043383781798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5181052478728816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.072102508792652e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5752660803846084e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5752655346877873e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.442951166092543e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7124242478748783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.712418972805608e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.275069270282984e-11, "numeric_tolerance_ratio": 5.260799777155296e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.8929631298524328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.8929516702191904e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1459633242338896e-10, "numeric_tolerance_ratio": 0.00011426576708166119 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.3614365343528334e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.3614378076454159e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2715614306756342e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4222289792087395e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4222278878150973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0898436361148734e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.5869325074600056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.586930688470602e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8124881342346815e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0005169383366592228, "latent_mse": 0.015044992789626122, "scaled_prediction": 7.486693357350305e-05, "scaled_weighted_latent": 0.00021789300080854446, "total": 0.0002927599416580051 }, "gradients": { "norms": { "prediction": 0.001070561120286584, "weighted_latent": 0.0012635241728276014, "total": 0.002058987971395254 }, "prediction_weighted_latent_dot": 7.48418131024664e-07, "prediction_weighted_latent_cosine": 0.5532854864566016, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0005865387502126396, "latent_mse": 0.014186912216246128, "scaled_prediction": 8.494699432048947e-05, "scaled_weighted_latent": 0.0002054656360996887, "total": 0.0002904126304201782 }, "gradients": { "norms": { "prediction": 0.001564753009006381, "weighted_latent": 0.0012265188852325082, "total": 0.0023790558334439993 }, "prediction_weighted_latent_dot": 8.535530469089281e-07, "prediction_weighted_latent_cosine": 0.44474439346736716, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.029072362929582596 }, { "step": 100, "losses": { "prediction_mse": 1.958247958100401e-05, "latent_mse": 2.6493105906411074e-05, "scaled_prediction": 2.8360832402540836e-06, "scaled_weighted_latent": 3.836932762624201e-07, "total": 3.219776544938213e-06 }, "gradients": { "norms": { "prediction": 2.248913369840011e-05, "weighted_latent": 5.7423247199039906e-05, "total": 5.3000701882410794e-05 }, "prediction_weighted_latent_dot": -4.970573952434165e-10, "prediction_weighted_latent_cosine": -0.38489836452138804, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005795139586552978 }, { "step": 200, "losses": { "prediction_mse": 2.0445553673198447e-05, "latent_mse": 8.523401447746437e-06, "scaled_prediction": 2.9610803267132724e-06, "scaled_weighted_latent": 1.2344236210992676e-07, "total": 3.0845226319797803e-06 }, "gradients": { "norms": { "prediction": 3.236097109038383e-05, "weighted_latent": 3.241108788643032e-05, "total": 2.005546775762923e-06 }, "prediction_weighted_latent_dot": -1.0468444999744975e-09, "prediction_weighted_latent_cosine": -0.9980838347441332, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00010220007970929146 }, { "step": 300, "losses": { "prediction_mse": 2.0447698261705227e-05, "latent_mse": 8.498044735461008e-06, "scaled_prediction": 2.961390919153928e-06, "scaled_weighted_latent": 1.2307513941323123e-07, "total": 3.084466015934595e-06 }, "gradients": { "norms": { "prediction": 3.2367966923629865e-05, "weighted_latent": 3.2362138881580904e-05, "total": 1.393101740632119e-07 }, "prediction_weighted_latent_dot": -1.0474870970611505e-09, "prediction_weighted_latent_cosine": -0.9999908889103759, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 6.84225233271718e-06 }, { "step": 400, "losses": { "prediction_mse": 2.0448296709218994e-05, "latent_mse": 8.49935440783156e-06, "scaled_prediction": 2.9614775485242717e-06, "scaled_weighted_latent": 1.230940966934213e-07, "total": 3.084571744693676e-06 }, "gradients": { "norms": { "prediction": 3.244182516937144e-05, "weighted_latent": 3.2366053346777335e-05, "total": 2.943585741377319e-06 }, "prediction_weighted_latent_dot": -1.045684538958369e-09, "prediction_weighted_latent_cosine": -0.9958769066096964, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00014213190297596157 }, { "step": 500, "losses": { "prediction_mse": 2.045622022706084e-05, "latent_mse": 8.50646938488353e-06, "scaled_prediction": 2.9626251034642337e-06, "scaled_weighted_latent": 1.2319713960096124e-07, "total": 3.0858222999086138e-06 }, "gradients": { "norms": { "prediction": 3.4209268051199615e-05, "weighted_latent": 3.236785778426565e-05, "total": 1.1325553714414127e-05 }, "prediction_weighted_latent_dot": -1.0448421017272835e-09, "prediction_weighted_latent_cosine": -0.9436108475923198, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0004590976459439844 }, { "step": 600, "losses": { "prediction_mse": 2.0449277144507505e-05, "latent_mse": 8.499263458361384e-06, "scaled_prediction": 2.9616194296977483e-06, "scaled_weighted_latent": 1.230927892947875e-07, "total": 3.0847122616251e-06 }, "gradients": { "norms": { "prediction": 3.28223904944025e-05, "weighted_latent": 3.2365216611651704e-05, "total": 5.815330951008946e-06 }, "prediction_weighted_latent_dot": -1.0454992427355592e-09, "prediction_weighted_latent_cosine": -0.9841810453139609, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0001859236799646169 }, { "step": 700, "losses": { "prediction_mse": 2.044854591076728e-05, "latent_mse": 8.503768185619265e-06, "scaled_prediction": 2.961513700938667e-06, "scaled_weighted_latent": 1.23158031328785e-07, "total": 3.084671789110871e-06 }, "gradients": { "norms": { "prediction": 3.274558548582718e-05, "weighted_latent": 3.2373885915149e-05, "total": 5.1326251195860095e-06 }, "prediction_weighted_latent_dot": -1.0469989319972228e-09, "prediction_weighted_latent_cosine": -0.987639945387094, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.00017646911146584898 }, { "step": 800, "losses": { "prediction_mse": 2.044757638941519e-05, "latent_mse": 8.501372576574795e-06, "scaled_prediction": 2.9613731840072433e-06, "scaled_weighted_latent": 1.2312332842157048e-07, "total": 3.0844964840071043e-06 }, "gradients": { "norms": { "prediction": 3.245994230383076e-05, "weighted_latent": 3.2369178370572627e-05, "total": 2.201377583332942e-06 }, "prediction_weighted_latent_dot": -1.0482827939028994e-09, "prediction_weighted_latent_cosine": -0.9976978541911432, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 6.59990546409972e-05 }, { "step": 900, "losses": { "prediction_mse": 2.045848304987885e-05, "latent_mse": 8.527510544809047e-06, "scaled_prediction": 2.9629527489305474e-06, "scaled_weighted_latent": 1.2350187716947403e-07, "total": 3.0864546261000214e-06 }, "gradients": { "norms": { "prediction": 3.636390465544537e-05, "weighted_latent": 3.24251304846257e-05, "total": 1.7051030226866715e-05 }, "prediction_weighted_latent_dot": -1.0414927809065944e-09, "prediction_weighted_latent_cosine": -0.8832914388944041, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005175215192139149 }, { "step": 1000, "losses": { "prediction_mse": 2.059229154838249e-05, "latent_mse": 8.529103979526553e-06, "scaled_prediction": 2.9823320346622495e-06, "scaled_weighted_latent": 1.2352495559753152e-07, "total": 3.1058571039466187e-06 }, "gradients": { "norms": { "prediction": 6.219880015123636e-05, "weighted_latent": 3.2423569791717455e-05, "total": 5.694122228305787e-05 }, "prediction_weighted_latent_dot": -8.38838110084339e-10, "prediction_weighted_latent_cosine": -0.41594443457749386, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0016192415496334434 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "d5fe1daf76ecb831a7fd5993adb2ac04798fc23dccbf307d6a53d4b2f3928d4f" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "ba6b4ea59eee224296331a02da33b214f683ba0badba648e3cce0f5fc96d7ad8" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "ef74077bb905c12696391663b55c343802e51762e1bba59d3ce8e162f3dc5f65" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "9f258f22a24d87544a4479eb1bbe15290aa56fda9ac504e3b048d91ba72e6031" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "10150b04c1102445168f5c8fa27b63c71953fdb606de561e746ae1f62fa5d70f" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "ed4a08762deffdb1ab5010c569dcc919502f9d92929e2be17560541ff8652636" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "78bec8d77fdb13b0d5243041a503a5ef5a06e388c4d7d0160e14a0b823200c2f" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "cb98d9ea7c73430260da2b8d7d02fa6c1539927fe235fafd8dea39a6224555fa" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "9e8d42808a1d2dd4e7f6b7883a03bc120faf26841f07430dd12eb08377904313" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "f1d47374e9acd7f83376c995e0b75802f2dee0a379ebc93ba9fb3fc05ab660d7" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "0690360799a01c879c995c1ef5b193fd5d7267c4b7bda1db7d496ae067e7b34d" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "a0d56a6c0cb366f7ef9baa918d85d54904469a947720c2e1064b3d1c99352424" } ], "final_losses": { "prediction_mse": 2.059229154838249e-05, "latent_mse": 8.529103979526553e-06, "scaled_prediction": 2.9823320346622495e-06, "scaled_weighted_latent": 1.2352495559753152e-07, "total": 3.1058571039466187e-06 }, "training_seconds": 3.331653399975039, "evaluation_seconds": 0.5156276000197977 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 4, "arm": "latent_only", "status": "passed", "files": { "seed-4-latent_only/result.json": "c25ddf4e97d248b46a51fff346dc71b9fcffa5f3a72d78fc72bd5244d0d7333e", "seed-4-latent_only/checkpoint.json": "2d931e11a6193b63dd4640446e85b1f406c5acc575a746081dcce32b3765c7c4", "inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "seed-4-latent_only/diagnostics-arrays.npz": "14d0e799adb468983669afdb48f041e71ba151ee58eb2ed41ad25a12e37578a9" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.0978193283081055e-07, "max_tolerance_ratio": 0.3144731636939299, "worst_tolerance_index": [ 0, 8 ], "actual_at_worst": 0.00018735229969024658, "saved_at_worst": 0.0001876726746559143, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450094886356965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.3450114895240404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.996207215045689e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800791404908523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 5.800786675536074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.702096589973028e-05 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2923264503479004, "saved_at_worst": 1.2923264503479004, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.48746048194662334, "worst_tolerance_index": [ 617, 0 ], "actual_at_worst": 0.0001264624297618866, "saved_at_worst": 0.0001259688287973404, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.437260263832286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372990295290947 }, "reported": { "count": 784, "observable_mse": 2.437260263832286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372990295290947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.8886492398451082e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030165999196469784 }, "reported": { "count": 580, "observable_mse": 1.888649057946168e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003016600152477622 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001788720871244953 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.8339056370896287e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00303729809820652 }, "reported": { "count": 112, "observable_mse": 2.833907274180092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372983310371637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017858810755260759 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 5.4130203352542594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003164678579196334 }, "reported": { "count": 92, "observable_mse": 5.413021062850021e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031646783463656902 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017686010818346055 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5274550353060476e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211733050644398 }, "reported": { "count": 98, "observable_mse": 2.5274570361943915e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171675249934 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012327307636645429 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0186165784252807e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "reported": { "count": 70, "observable_mse": 2.0186189431115054e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211709767580032 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002817670465808772 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1840363842784427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171209588647 }, "reported": { "count": 14, "observable_mse": 2.1840449335286394e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221172373741865 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0008805219275269677 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.415066334535368e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221169812604785 }, "reported": { "count": 14, "observable_mse": 5.415061968960799e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.003698191444345532 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493371140328236e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220667831599712 }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.008045871341972934 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9936096578021534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022065619006753 }, "reported": { "count": 70, "observable_mse": 1.9936062017222866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220558401197195 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.778887033462524e-09, "numeric_tolerance_ratio": 0.007509480187709456 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.3793707441654988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022068180143833 }, "reported": { "count": 14, "observable_mse": 2.3793658328941092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009118654187569325 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.497939855558798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022070275619626 }, "reported": { "count": 14, "observable_mse": 5.497950769495219e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220600310713053 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.007867071950464247 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5550732971169055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836770940572023 }, "reported": { "count": 98, "observable_mse": 2.55507566180313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836845446377993 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.005651364435865696 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.939376124937553e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836775597184896 }, "reported": { "count": 70, "observable_mse": 1.9393763068364933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836838461458683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004768338995395631 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.679098724911455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836782582104206 }, "reported": { "count": 14, "observable_mse": 2.679110002645757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836875714361668 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.007064203922984993 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.509533730219118e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836743000894785 }, "reported": { "count": 14, "observable_mse": 5.509537731995806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003183686174452305 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009006860956200402 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5143033781205304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363468684256077 }, "reported": { "count": 98, "observable_mse": 2.514301559131127e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634756691753864 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.005226873219612861 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949908801179845e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588928636163473 }, "reported": { "count": 72, "observable_mse": 1.949909346876666e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588993828743696 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004880086177358148 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.875245627365075e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634670544415712 }, "reported": { "count": 14, "observable_mse": 2.8752372600138187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363472642377019 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004181499522790897 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.4795717005617917e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033907967153936625 }, "reported": { "count": 12, "observable_mse": 5.4795644246041775e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003390809055417776 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.00921529390986458 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5130846552201547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 98, "observable_mse": 2.5130850190180354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750871662050486 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0031334013256818006 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949693978531286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003370406571775675 }, "reported": { "count": 72, "observable_mse": 1.949694888025988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033704107627272606 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.003134497256772981 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750897273421288 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.752088665962219e-09, "numeric_tolerance_ratio": 0.005048256724707581 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.473628334584646e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003403007285669446 }, "reported": { "count": 12, "observable_mse": 5.4736312449676916e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034030096139758825 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0017371519558643553 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509041734039783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103164955973625 }, "reported": { "count": 98, "observable_mse": 2.5509052647976205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103166352957487 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0010661422012763408 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9929024347220547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031647231429815 }, "reported": { "count": 70, "observable_mse": 1.9929013433284126e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031661201268435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.001066142220220661 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.406693783996161e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103160299360752 }, "reported": { "count": 14, "observable_mse": 2.406714338576421e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031658872962 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004264568956659927 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.485124347615056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103169146925211 }, "reported": { "count": 14, "observable_mse": 5.485117799253203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031689140945673 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001776903321481438 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386959456838667e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.3563443107832427e-07 }, "reported": { "count": 98, "observable_mse": 2.8386977646732703e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.3200118448585272e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.6332465924715507e-09, "numeric_tolerance_ratio": 0.003633162302675786 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450111257261597e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.3472955490433378e-07 }, "reported": { "count": 84, "observable_mse": 2.3450154913007282e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.3212984956444416e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.599705339889624e-09, "numeric_tolerance_ratio": 0.0025996449943694777 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800804137834348e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.4099242068587046e-07 }, "reported": { "count": 14, "observable_mse": 5.800792860100046e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 2.312278155613967e-07 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.764605124473746e-09, "numeric_tolerance_ratio": 0.009764379344863126 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492266927845776e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384734462946653 }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006583761406554621 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.54845256981207e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003532622940838337 }, "reported": { "count": 72, "observable_mse": 8.548483492631931e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035326124634593725 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.0077423180429610685 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4730835573573131e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003438472282141447 }, "reported": { "count": 14, "observable_mse": 1.4730812836205587e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.683411240577698e-09, "numeric_tolerance_ratio": 0.005717477010955329 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.987682041246444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080525808036327 }, "reported": { "count": 12, "observable_mse": 4.987671127310023e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080549091100693 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0018178454519917982 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2935075759887695, "saved_at_worst": 1.2935075759887695, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.5794924900882756, "worst_tolerance_index": [ 5875, 12 ], "actual_at_worst": 0.0005421936511993408, "saved_at_worst": 0.0005428045988082886, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.416123970760964e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005598969757556915 }, "reported": { "count": 6272, "observable_mse": 2.4161243345588446e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005598969291895628 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.656612873077393e-10, "numeric_tolerance_ratio": 0.0002985205487580974 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.8748192815110087e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055619413033127785 }, "reported": { "count": 4092, "observable_mse": 1.874819463409949e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0055619413033127785 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-12, "numeric_tolerance_ratio": 1.8155855084970412e-06 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.7032006983063184e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005598968360573053 }, "reported": { "count": 896, "observable_mse": 2.7032017896999605e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005598968360573053 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0884513385212222e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.9408889279002324e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005715375766158104 }, "reported": { "count": 1284, "observable_mse": 3.940889655495994e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.005715375766158104 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.247396424584503e-06 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2881712913513184, "saved_at_worst": 1.2881712913513184, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4369506557359455, "worst_tolerance_index": [ 2489, 7 ], "actual_at_worst": 0.0009123608469963074, "saved_at_worst": 0.0009128376841545105, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.416784238652326e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010839683003723621 }, "reported": { "count": 3136, "observable_mse": 3.416786785237491e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010839683003723621 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5379136551252936e-05 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 2.91789820039412e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01080263126641512 }, "reported": { "count": 1582, "observable_mse": 2.9179005650803447e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01080263126641512 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.357806380040942e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.484251647023484e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010839683935046196 }, "reported": { "count": 448, "observable_mse": 3.484255285002291e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010839683935046196 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6253471720475385e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 4.1030518332263455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01089246105402708 }, "reported": { "count": 1106, "observable_mse": 4.1030543798115104e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01089246105402708 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.546585164964199e-11, "numeric_tolerance_ratio": 2.5361790842645216e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2869694232940674, "saved_at_worst": 1.2869694232940674, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4155178327569591, "worst_tolerance_index": [ 2367, 8 ], "actual_at_worst": -0.0002209991216659546, "saved_at_worst": -0.00022057443857192993, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 8.863356924848631e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020777098834514618 }, "reported": { "count": 3136, "observable_mse": 8.863364200806245e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02077709697186947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006052049518944008 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 7.996758358785883e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02014041878283024 }, "reported": { "count": 1001, "observable_mse": 7.996763451956213e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02014041878283024 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.093170329928398e-11, "numeric_tolerance_ratio": 5.052764566908406e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 8.91760500962846e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02077709697186947 }, "reported": { "count": 448, "observable_mse": 8.91761010279879e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020777098834514618 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006052049152670997 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 9.363156277686357e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.021145818755030632 }, "reported": { "count": 1687, "observable_mse": 9.363165736431256e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.021145818755030632 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.371002647533067e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2679539918899536, "saved_at_worst": 1.2679539918899536, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.42896605801972054, "worst_tolerance_index": [ 3103, 0 ], "actual_at_worst": -0.0004641897976398468, "saved_at_worst": -0.00046463869512081146, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0003656770568341017, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043556950986385345 }, "reported": { "count": 3136, "observable_mse": 0.0003656771732494235, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043556950986385345 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-10, "numeric_tolerance_ratio": 0.00011230845788576787 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.0003249439469072968, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04104834794998169 }, "reported": { "count": 407, "observable_mse": 0.0003249441215302795, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04104834794998169 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016912729084534847 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0003670162695925683, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043556950986385345 }, "reported": { "count": 448, "observable_mse": 0.000367016444215551, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043556954711675644 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.0006955754520616506 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.00037268202868290246, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043989527970552444 }, "reported": { "count": 2281, "observable_mse": 0.00037268217420205474, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.043989527970552444 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014029076553178296 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006257921922951937 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5511306375847198e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5511330022709444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3586689464189724e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006255257874727249 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5392424504389055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5392444513272494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9958204678516046e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006481279619038105 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5999313947977498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5999332137871534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8142724164315183e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006476297043263912 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.6249348593410105e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.624933586048428e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2699590243860832e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.021607279777526855 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.766223668819293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.7661858338397e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.7834979593753815e-10, "numeric_tolerance_ratio": 0.00037580701583076647 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.019717823714017868 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.143248901935294e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.143239443190396e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.40099235142008e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020352382212877274 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.072931475704536e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.072944572428241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00013004741886984787 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012590174563229084 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.74880874005612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.748832386918366e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-10, "numeric_tolerance_ratio": 0.000235585452088286 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 3.657481784102856e-07 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.838700129359495e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.838699583662674e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.4415213662008435e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0059067197144031525 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7997512006550096e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.7997439246973954e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.255643669898858e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0037216152995824814 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5795047147548757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.579504350956995e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6322416655772068e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492263289866969e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.185452315956354e-12, "numeric_tolerance_ratio": 8.17360690293936e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006914579775184393 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1585754186380655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.158584877382964e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.428962719383602e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493387511232868e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.447291564678985e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5946865207515657e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.594685065560043e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4514255306884354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5181043383781798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5181052478728816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.072102508792652e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509087208774872e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5509103579679504e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6329250178494538e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.676446456462145e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.6764359063236043e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0550138540565968e-10, "numeric_tolerance_ratio": 0.00010521977143134568 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.8929631298524328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.8929516702191904e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1459633242338896e-10, "numeric_tolerance_ratio": 0.00011426576708166119 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.3614365343528334e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.3614378076454159e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2715614306756342e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4222289792087395e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4222278878150973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0898436361148734e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.5869325074600056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.586930688470602e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8124881342346815e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 1000, "optimizer_initial_state": { "type": "Adam", "parameter_shape": [ 10, 11 ], "learning_rate": 0.003, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.0, "foreach": false, "fused": false, "fresh": true, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 0.0005169383366592228, "latent_mse": 0.015044992789626122, "scaled_prediction": 7.486693357350305e-05, "scaled_weighted_latent": 0.00021789300080854446, "total": 0.00021789300080854446 }, "gradients": { "norms": { "prediction": 0.001070561120286584, "weighted_latent": 0.0012635241728276014, "total": 0.0012635241728276014 }, "prediction_weighted_latent_dot": 7.48418131024664e-07, "prediction_weighted_latent_cosine": 0.5532854864566016, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 }, { "step": 1, "losses": { "prediction_mse": 0.0005966797471046448, "latent_mse": 0.014187210239470005, "scaled_prediction": 8.641569002065808e-05, "scaled_weighted_latent": 0.0002054699434665963, "total": 0.0002054699434665963 }, "gradients": { "norms": { "prediction": 0.0015940532321110368, "weighted_latent": 0.0012265965342521667, "total": 0.0012265965342521667 }, "prediction_weighted_latent_dot": 8.696779332240112e-07, "prediction_weighted_latent_cosine": 0.4447888555205473, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.03141213580965996 }, { "step": 100, "losses": { "prediction_mse": 2.1970061425236054e-05, "latent_mse": 1.11036715679802e-05, "scaled_prediction": 3.181871079505072e-06, "scaled_weighted_latent": 1.6081179410321056e-07, "total": 1.6081179410321056e-07 }, "gradients": { "norms": { "prediction": 0.00014432429452426732, "weighted_latent": 3.859043135889806e-05, "total": 3.859043135889806e-05 }, "prediction_weighted_latent_dot": 1.7062163903958094e-09, "prediction_weighted_latent_cosine": 0.3063479886041389, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0005903447163291276 }, { "step": 200, "losses": { "prediction_mse": 2.346691871935036e-05, "latent_mse": 1.3682244226487228e-10, "scaled_prediction": 3.3986573271249654e-06, "scaled_weighted_latent": 1.981566386671796e-12, "total": 1.981566386671796e-12 }, "gradients": { "norms": { "prediction": 0.0001162839907919988, "weighted_latent": 1.2867796783666563e-07, "total": 1.2867796783666563e-07 }, "prediction_weighted_latent_dot": 1.2082485185971326e-11, "prediction_weighted_latent_cosine": 0.8074806977710687, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.09139659293578e-06 }, { "step": 300, "losses": { "prediction_mse": 2.3450125809176825e-05, "latent_mse": 5.86610092640806e-14, "scaled_prediction": 3.396225110918749e-06, "scaled_weighted_latent": 8.495732520026752e-16, "total": 8.495732520026752e-16 }, "gradients": { "norms": { "prediction": 0.0001155613936134614, "weighted_latent": 2.045127178718076e-09, "total": 2.045127178718076e-09 }, "prediction_weighted_latent_dot": 4.812350774222474e-15, "prediction_weighted_latent_cosine": 0.020362175901058977, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.1458557480636955e-07 }, { "step": 400, "losses": { "prediction_mse": 2.3450151275028475e-05, "latent_mse": 6.174448024262938e-14, "scaled_prediction": 3.3962289762712317e-06, "scaled_weighted_latent": 8.942304701082572e-16, "total": 8.942304701082572e-16 }, "gradients": { "norms": { "prediction": 0.00011556194658624008, "weighted_latent": 2.1368944391753075e-09, "total": 2.1368944391753075e-09 }, "prediction_weighted_latent_dot": 7.334172840171762e-15, "prediction_weighted_latent_cosine": 0.029699779355648236, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 1.903740951547661e-07 }, { "step": 500, "losses": { "prediction_mse": 2.345011853321921e-05, "latent_mse": 4.6259938696509295e-14, "scaled_prediction": 3.3962242014240474e-06, "scaled_weighted_latent": 6.699715622404255e-16, "total": 6.699715622404255e-16 }, "gradients": { "norms": { "prediction": 0.00011556225217645988, "weighted_latent": 1.8800572210153632e-09, "total": 1.8800572210153632e-09 }, "prediction_weighted_latent_dot": 7.533627923426468e-15, "prediction_weighted_latent_cosine": 0.034675050513547474, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.09897160377659e-07 }, { "step": 600, "losses": { "prediction_mse": 2.3450134904123843e-05, "latent_mse": 5.471406273470543e-14, "scaled_prediction": 3.3962264751608018e-06, "scaled_weighted_latent": 7.924105982825083e-16, "total": 7.924105982825083e-16 }, "gradients": { "norms": { "prediction": 0.0001155606732936576, "weighted_latent": 2.391824294178946e-09, "total": 2.391824294178946e-09 }, "prediction_weighted_latent_dot": -8.797817820939934e-16, "prediction_weighted_latent_cosine": -0.003182992595752107, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 2.5450245288993756e-07 }, { "step": 700, "losses": { "prediction_mse": 2.345011853321921e-05, "latent_mse": 7.094315640585741e-14, "scaled_prediction": 3.3962242014240474e-06, "scaled_weighted_latent": 1.0274526590853608e-15, "total": 1.0274526590853608e-15 }, "gradients": { "norms": { "prediction": 0.00011556035315152258, "weighted_latent": 3.254916558503851e-09, "total": 3.254916558503851e-09 }, "prediction_weighted_latent_dot": 3.4637555258227757e-15, "prediction_weighted_latent_cosine": 0.009208703960344337, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.699499752154225e-07 }, { "step": 800, "losses": { "prediction_mse": 2.3450140361092053e-05, "latent_mse": 4.264028230982139e-14, "scaled_prediction": 3.3962273846555036e-06, "scaled_weighted_latent": 6.175489048930937e-16, "total": 6.175489048930937e-16 }, "gradients": { "norms": { "prediction": 0.00011556156096048653, "weighted_latent": 2.075037919269107e-09, "total": 2.075037919269107e-09 }, "prediction_weighted_latent_dot": -1.0912143347393492e-15, "prediction_weighted_latent_cosine": -0.004550620569282825, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 3.335407825488801e-07 }, { "step": 900, "losses": { "prediction_mse": 2.345009306736756e-05, "latent_mse": 2.7154482072741688e-14, "scaled_prediction": 3.3962205634452403e-06, "scaled_weighted_latent": 3.932718122866757e-16, "total": 3.932718122866757e-16 }, "gradients": { "norms": { "prediction": 0.0001155607751570642, "weighted_latent": 1.4901179179460655e-09, "total": 1.4901179179460655e-09 }, "prediction_weighted_latent_dot": 4.4029780567485764e-15, "prediction_weighted_latent_cosine": 0.02556909977134831, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 5.495569439517567e-07 }, { "step": 1000, "losses": { "prediction_mse": 2.3450154913007282e-05, "latent_mse": 5.388426521386186e-14, "scaled_prediction": 3.3962294310185825e-06, "scaled_weighted_latent": 7.803928418873051e-16, "total": 7.803928418873051e-16 }, "gradients": { "norms": { "prediction": 0.00011556116078281775, "weighted_latent": 2.6767299488028584e-09, "total": 2.6767299488028584e-09 }, "prediction_weighted_latent_dot": 1.5839785470132643e-14, "prediction_weighted_latent_cosine": 0.05120741368883312, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 9.743667988004745e-07 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "f699effcd25959f7680c911e7e747dadfe1905be0a36d1db90898b83d30d32e2" }, { "step": 1, "path": "checkpoints/step-0001.json", "sha256": "16d6290bd18f6fc09a6d359f42c2850fd7be6d053ff3733c29ed632302aed6ac" }, { "step": 100, "path": "checkpoints/step-0100.json", "sha256": "79468a7b7bb60cb31ae067bd466365ae6ce7ae479a6c5c974029a0277487ea75" }, { "step": 200, "path": "checkpoints/step-0200.json", "sha256": "67590f7564bf109fc28336dca4bb0714de4e6b7975593730f38566feee20fdcc" }, { "step": 300, "path": "checkpoints/step-0300.json", "sha256": "d2d1d7f10f51fd7867f9870527fd6b4a75abf7f76da66c4a162b05fbfc7b3a74" }, { "step": 400, "path": "checkpoints/step-0400.json", "sha256": "96b08ccd99469c3263897cd45890578b23fbe119971ba0f9636425b26ac97a06" }, { "step": 500, "path": "checkpoints/step-0500.json", "sha256": "b5bac88343b6003c810ddaeee3592e4f0f114dc48d28e324ad13a336fc8b57b0" }, { "step": 600, "path": "checkpoints/step-0600.json", "sha256": "e75779ee7f9a7ead20804c9d3654a7014174f133e1fefde530cd3552af26f5e4" }, { "step": 700, "path": "checkpoints/step-0700.json", "sha256": "7dae43d65fefd08534539ce218529e26f7e99234942b39ee4122e3e31d921e9f" }, { "step": 800, "path": "checkpoints/step-0800.json", "sha256": "db64aa038ad79127161d8cf6c8bebea01282a7973c6b4b473129af617d6374f5" }, { "step": 900, "path": "checkpoints/step-0900.json", "sha256": "59723d6858c9144b10a158ec319c65303b53ff752beb7888c02a7910687653f8" }, { "step": 1000, "path": "checkpoints/step-1000.json", "sha256": "2d931e11a6193b63dd4640446e85b1f406c5acc575a746081dcce32b3765c7c4" } ], "final_losses": { "prediction_mse": 2.3450154913007282e-05, "latent_mse": 5.388426521386186e-14, "scaled_prediction": 3.3962294310185825e-06, "scaled_weighted_latent": 7.803928418873051e-16, "total": 7.803928418873051e-16 }, "training_seconds": 2.835771100013517, "evaluation_seconds": 0.4971197000122629 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." }, { "seed": 4, "arm": "refit", "status": "passed", "files": { "seed-4-refit/result.json": "bbeefd997166ae9237853a96b43f56fb313eddeba42dfb979793b79d0007ae88", "seed-4-refit/checkpoint.json": "54d0f37cf21d2f3971d6eede53623bef7794973f5523f1f4b98bab3c2bb4a858", "inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "seed-4-refit/diagnostics-arrays.npz": "e1c92542d749f49d0bb2b5de987d06fb43bcfc8239d0efffa527ec14fff421f8" }, "provenance": { "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df", "design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9", "parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454", "parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57", "recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314" }, "frozen_tensor_isolation": "passed", "encoding": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "cases": [ { "set": "reconstruction", "count": 98, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 98, 10 ], "compared_scalars": 980, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.29099440574646, "saved_at_worst": 1.29099440574646, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 98, 16 ], "compared_scalars": 1568, "outside_tolerance": 0, "max_absolute_error": 4.0978193283081055e-07, "max_tolerance_ratio": 0.3144731636939299, "worst_tolerance_index": [ 0, 8 ], "actual_at_worst": 0.00018735229969024658, "saved_at_worst": 0.0001876726746559143, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 98, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 }, { "case": "seen", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450094886356965e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 84, "observable_mse": 2.3450114895240404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.996207215045689e-05 }, "comparisons": 6 }, { "case": "held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800791404908523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 14, "observable_mse": 5.800786675536074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.702096589973028e-05 }, "comparisons": 6 } ], "saved_diagnostic_differences": [] }, { "set": "one_step", "count": 784, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 784, 10 ], "compared_scalars": 7840, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2923264503479004, "saved_at_worst": 1.2923264503479004, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 784, 16 ], "compared_scalars": 12544, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.37291832992681323, "worst_tolerance_index": [ 343, 9 ], "actual_at_worst": 0.001188725233078003, "saved_at_worst": 0.0011883080005645752, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 784, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 784, "observable_mse": 2.437259718135465e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372990295290947 }, "reported": { "count": 784, "observable_mse": 2.437259718135465e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372990295290947 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 580, "observable_mse": 1.8886488760472275e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030165999196469784 }, "reported": { "count": 580, "observable_mse": 1.8886488760472275e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003016600152477622 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001788720871244953 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 112, "observable_mse": 2.8339039999991655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.00303729809820652 }, "reported": { "count": 112, "observable_mse": 2.833904727594927e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030372983310371637 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017858810755260759 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 92, "observable_mse": 5.4130203352542594e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003164678579196334 }, "reported": { "count": 92, "observable_mse": 5.413021062850021e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031646783463656902 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.00017686010818346055 }, "comparisons": 7 }, { "case": "action_X/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5274550353060476e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211733050644398 }, "reported": { "count": 98, "observable_mse": 2.5274570361943915e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171675249934 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6298145055770874e-09, "numeric_tolerance_ratio": 0.0012327307636645429 }, "comparisons": 7 }, { "case": "action_X/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 2.0186165784252807e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "reported": { "count": 70, "observable_mse": 2.0186189431115054e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211709767580032 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.725290298461914e-09, "numeric_tolerance_ratio": 0.002817670465808772 }, "comparisons": 7 }, { "case": "action_X/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.1840363842784427e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221171209588647 }, "reported": { "count": 14, "observable_mse": 2.1840449335286394e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221172373741865 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1641532182693481e-09, "numeric_tolerance_ratio": 0.0008805219275269677 }, "comparisons": 7 }, { "case": "action_X/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.415066334535368e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003221169812604785 }, "reported": { "count": 14, "observable_mse": 5.415061968960799e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0032211747020483017 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.889443516731262e-09, "numeric_tolerance_ratio": 0.003698191444345532 }, "comparisons": 7 }, { "case": "action_Y/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493371140328236e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220667831599712 }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.008045871341972934 }, "comparisons": 7 }, { "case": "action_Y/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9936096578021534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022065619006753 }, "reported": { "count": 70, "observable_mse": 1.9936062017222866e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220558401197195 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.778887033462524e-09, "numeric_tolerance_ratio": 0.007509480187709456 }, "comparisons": 7 }, { "case": "action_Y/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.3793707441654988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022068180143833 }, "reported": { "count": 14, "observable_mse": 2.3793658328941092e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022056305781007 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009118654187569325 }, "comparisons": 7 }, { "case": "action_Y/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.497939855558798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003022070275619626 }, "reported": { "count": 14, "observable_mse": 5.497950769495219e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0030220600310713053 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0244548320770264e-08, "numeric_tolerance_ratio": 0.007867071950464247 }, "comparisons": 7 }, { "case": "action_Xinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5550732971169055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836770940572023 }, "reported": { "count": 98, "observable_mse": 2.55507566180313e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836845446377993 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.450580596923828e-09, "numeric_tolerance_ratio": 0.005651364435865696 }, "comparisons": 7 }, { "case": "action_Xinv/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.939376124937553e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836775597184896 }, "reported": { "count": 70, "observable_mse": 1.9393763068364933e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836838461458683 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.28642737865448e-09, "numeric_tolerance_ratio": 0.004768338995395631 }, "comparisons": 7 }, { "case": "action_Xinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.679098724911455e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836782582104206 }, "reported": { "count": 14, "observable_mse": 2.679110002645757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836875714361668 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-09, "numeric_tolerance_ratio": 0.007064203922984993 }, "comparisons": 7 }, { "case": "action_Xinv/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.509533730219118e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031836743000894785 }, "reported": { "count": 14, "observable_mse": 5.509537731995806e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003183686174452305 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1874362826347351e-08, "numeric_tolerance_ratio": 0.009006860956200402 }, "comparisons": 7 }, { "case": "action_R/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5143033781205304e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363468684256077 }, "reported": { "count": 98, "observable_mse": 2.514301559131127e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634756691753864 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.984919309616089e-09, "numeric_tolerance_ratio": 0.005226873219612861 }, "comparisons": 7 }, { "case": "action_R/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949908801179845e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588928636163473 }, "reported": { "count": 72, "observable_mse": 1.949909346876666e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033588993828743696 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.51925802230835e-09, "numeric_tolerance_ratio": 0.004880086177358148 }, "comparisons": 7 }, { "case": "action_R/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.875245627365075e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033634670544415712 }, "reported": { "count": 14, "observable_mse": 2.8752372600138187e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003363472642377019 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004181499522790897 }, "comparisons": 7 }, { "case": "action_R/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.4795717005617917e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033907967153936625 }, "reported": { "count": 12, "observable_mse": 5.4795644246041775e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003390809055417776 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.234002411365509e-08, "numeric_tolerance_ratio": 0.00921529390986458 }, "comparisons": 7 }, { "case": "action_Rinv/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5130846552201547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 98, "observable_mse": 2.5130850190180354e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750871662050486 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.0031334013256818006 }, "comparisons": 7 }, { "case": "action_Rinv/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 1.949693978531286e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003370406571775675 }, "reported": { "count": 72, "observable_mse": 1.949694888025988e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033704107627272606 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.190951585769653e-09, "numeric_tolerance_ratio": 0.003134497256772981 }, "comparisons": 7 }, { "case": "action_Rinv/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003375082975253463 }, "reported": { "count": 14, "observable_mse": 2.872911500162445e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0033750897273421288 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 6.752088665962219e-09, "numeric_tolerance_ratio": 0.005048256724707581 }, "comparisons": 7 }, { "case": "action_Rinv/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 5.473628334584646e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003403007285669446 }, "reported": { "count": 12, "observable_mse": 5.4736312449676916e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034030096139758825 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0017371519558643553 }, "comparisons": 7 }, { "case": "action_S/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509041734039783e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103164955973625 }, "reported": { "count": 98, "observable_mse": 2.5509052647976205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103166352957487 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.0010661422012763408 }, "comparisons": 7 }, { "case": "action_S/seen_stays_in_support", "status": "passed", "actual": { "count": 70, "observable_mse": 1.9929024347220547e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031647231429815 }, "reported": { "count": 70, "observable_mse": 1.9929013433284126e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031661201268435 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3969838619232178e-09, "numeric_tolerance_ratio": 0.001066142220220661 }, "comparisons": 7 }, { "case": "action_S/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 2.406693783996161e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103160299360752 }, "reported": { "count": 14, "observable_mse": 2.406714338576421e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031658872962 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.587935447692871e-09, "numeric_tolerance_ratio": 0.004264568956659927 }, "comparisons": 7 }, { "case": "action_S/seen_enters_held", "status": "passed", "actual": { "count": 14, "observable_mse": 5.485124347615056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003103169146925211 }, "reported": { "count": 14, "observable_mse": 5.485117799253203e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0031031689140945673 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-10, "numeric_tolerance_ratio": 0.0001776903321481438 }, "comparisons": 7 }, { "case": "action_H/all", "status": "passed", "actual": { "count": 98, "observable_mse": 2.838693035300821e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.229526359206479e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6276809331726797e-06 }, "comparisons": 7 }, { "case": "action_H/seen_stays_in_support", "status": "passed", "actual": { "count": 84, "observable_mse": 2.3450098524335772e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "reported": { "count": 84, "observable_mse": 2.3450114895240404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 5.613619272490454e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.6370904631912708e-11, "numeric_tolerance_ratio": 1.6332604486737455e-05 }, "comparisons": 7 }, { "case": "action_H/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 5.800791404908523e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360026644082518e-16 }, "reported": { "count": 14, "observable_mse": 5.800786675536074e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 1.5360023997104558e-16 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.729372449219227e-11, "numeric_tolerance_ratio": 4.702096589973028e-05 }, "comparisons": 7 }, { "case": "action_H/seen_enters_held", "status": "passed", "actual": { "count": 0, "status": "empty" }, "reported": { "count": 0, "status": "empty" }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 1 }, { "case": "action_P/all", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492266927845776e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384734462946653 }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.847564458847046e-09, "numeric_tolerance_ratio": 0.006583761406554621 }, "comparisons": 7 }, { "case": "action_P/seen_stays_in_support", "status": "passed", "actual": { "count": 72, "observable_mse": 8.54845256981207e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003532622940838337 }, "reported": { "count": 72, "observable_mse": 8.548483492631931e-06, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0035326124634593725 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0477378964424133e-08, "numeric_tolerance_ratio": 0.0077423180429610685 }, "comparisons": 7 }, { "case": "action_P/held_initial", "status": "passed", "actual": { "count": 14, "observable_mse": 1.4730835573573131e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.003438472282141447 }, "reported": { "count": 14, "observable_mse": 1.4730812836205587e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0034384645987302065 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.683411240577698e-09, "numeric_tolerance_ratio": 0.005717477010955329 }, "comparisons": 7 }, { "case": "action_P/seen_enters_held", "status": "passed", "actual": { "count": 12, "observable_mse": 4.987682041246444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080525808036327 }, "reported": { "count": 12, "observable_mse": 4.987671127310023e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.0028080549091100693 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3283064365386963e-09, "numeric_tolerance_ratio": 0.0018178454519917982 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_2", "count": 6272, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 6272, 10 ], "compared_scalars": 62720, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2935075759887695, "saved_at_worst": 1.2935075759887695, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 6272, 16 ], "compared_scalars": 100352, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.5794924900882756, "worst_tolerance_index": [ 5875, 12 ], "actual_at_worst": 0.0005421936511993408, "saved_at_worst": 0.0005428045988082886, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 6272, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 6272, "observable_mse": 2.4130684323608875e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005596357863396406 }, "reported": { "count": 6272, "observable_mse": 2.413069159956649e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005596357863396406 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.258442490460377e-06 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 4092, "observable_mse": 1.8717970306170173e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005558918230235577 }, "reported": { "count": 4092, "observable_mse": 1.8717970306170173e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005558918230235577 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 0.0, "numeric_tolerance_ratio": 0.0 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 896, "observable_mse": 2.6992713173967786e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005596356466412544 }, "reported": { "count": 896, "observable_mse": 2.6992727725883015e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005596356466412544 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4512741380702305e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1284, "observable_mse": 3.938337249564938e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.005714038852602243 }, "reported": { "count": 1284, "observable_mse": 3.938339068554342e-05, "exact_state_accuracy": 0.9999999403953552, "per_field_accuracy": { "x": 0.9999999403953552, "y": 0.9999999403953552, "h": 0.9999999403953552 }, "normalized_latent_rmse": 0.005714038852602243 }, "differences": [], "max_errors": { "accuracy_abs": 5.960464477539063e-08, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8118537092960308e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_4", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2876759767532349, "saved_at_worst": 1.2876759767532349, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 7.152557373046875e-07, "max_tolerance_ratio": 0.4632957891476394, "worst_tolerance_index": [ 655, 8 ], "actual_at_worst": 0.0014174729585647583, "saved_at_worst": 0.00141800194978714, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 3.346535959281027e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010823823511600494 }, "reported": { "count": 3136, "observable_mse": 3.3465381420683116e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.01082382258027792 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.313225746154785e-10, "numeric_tolerance_ratio": 0.0004472390076438352 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1582, "observable_mse": 2.8475007638917305e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010785087011754513 }, "reported": { "count": 1582, "observable_mse": 2.8475038561737165e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010785087011754513 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.092281986027956e-11, "numeric_tolerance_ratio": 3.0835017030380373e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 3.412026126170531e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010823823511600494 }, "reported": { "count": 448, "observable_mse": 3.412029764149338e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010823823511600494 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-11, "numeric_tolerance_ratio": 3.6256081242586016e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1106, "observable_mse": 4.033817822346464e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010878991335630417 }, "reported": { "count": 1106, "observable_mse": 4.033818913740106e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.010878991335630417 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0870088452879884e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_8", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2870361804962158, "saved_at_worst": 1.2870361804962158, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.4524213331222226, "worst_tolerance_index": [ 206, 7 ], "actual_at_worst": -0.00037545710802078247, "saved_at_worst": -0.00037498772144317627, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 8.535668166587129e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020744062960147858 }, "reported": { "count": 3136, "observable_mse": 8.535675442544743e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.02074406109750271 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.862645149230957e-09, "numeric_tolerance_ratio": 0.0006058552717949994 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 1001, "observable_mse": 7.651780470041558e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020098954439163208 }, "reported": { "count": 1001, "observable_mse": 7.651787745999172e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020098954439163208 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-11, "numeric_tolerance_ratio": 7.220706302182925e-05 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 8.586674084654078e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020744062960147858 }, "reported": { "count": 448, "observable_mse": 8.586679905420169e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.020744062960147858 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.820766091346741e-11, "numeric_tolerance_ratio": 5.771210553655716e-05 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 1687, "observable_mse": 9.046589548233896e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.021117528900504112 }, "reported": { "count": 1687, "observable_mse": 9.046597551787272e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.021117528900504112 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.003553375601768e-11, "numeric_tolerance_ratio": 7.931797594898488e-05 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] }, { "set": "composition_16", "count": 3136, "status": "passed", "continuous": { "latent": { "status": "passed", "shape": [ 3136, 10 ], "compared_scalars": 31360, "outside_tolerance": 0, "max_absolute_error": 0.0, "max_tolerance_ratio": 0.0, "worst_tolerance_index": [ 0, 0 ], "actual_at_worst": 1.2679535150527954, "saved_at_worst": 1.2679535150527954, "witnesses": [] }, "logits": { "status": "passed", "shape": [ 3136, 16 ], "compared_scalars": 50176, "outside_tolerance": 0, "max_absolute_error": 5.960464477539062e-07, "max_tolerance_ratio": 0.42896605801972054, "worst_tolerance_index": [ 3103, 0 ], "actual_at_worst": -0.0004641897976398468, "saved_at_worst": -0.00046463869512081146, "witnesses": [] } }, "categorical": { "status": "passed", "compared_predictions": 3136, "differing_predictions": 0, "cases": [] }, "metric_comparisons": [ { "case": "all", "status": "passed", "actual": { "count": 3136, "observable_mse": 0.0003546668740455061, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04347893223166466 }, "reported": { "count": 3136, "observable_mse": 0.00035466704866848886, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04347893223166466 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016864181331919994 }, "comparisons": 7 }, { "case": "seen_stays_in_support", "status": "passed", "actual": { "count": 407, "observable_mse": 0.00031717080855742097, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.040965866297483444 }, "reported": { "count": 407, "observable_mse": 0.00031717095407657325, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.040965866297483444 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014104559566900483 }, "comparisons": 7 }, { "case": "held_initial", "status": "passed", "actual": { "count": 448, "observable_mse": 0.0003559494798537344, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04347892850637436 }, "reported": { "count": 448, "observable_mse": 0.0003559496544767171, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04347892850637436 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.7462298274040222e-10, "numeric_tolerance_ratio": 0.00016862092668142262 }, "comparisons": 7 }, { "case": "seen_enters_held", "status": "passed", "actual": { "count": 2281, "observable_mse": 0.00036110542714595795, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04391222447156906 }, "reported": { "count": 2281, "observable_mse": 0.00036110557266511023, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 }, "normalized_latent_rmse": 0.04391222447156906 }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-10, "numeric_tolerance_ratio": 0.00014044751427644966 }, "comparisons": 7 } ], "saved_diagnostic_differences": [] } ], "relations": [ { "relation": "X_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006257921922951937 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5511306375847198e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5511330022709444e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-11, "numeric_tolerance_ratio": 2.3586689464189724e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_X", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006255257874727249 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5392424504389055e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5392444513272494e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.000888343900442e-11, "numeric_tolerance_ratio": 1.9958204678516046e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006481279619038105 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5999313947977498e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5999332137871534e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8142724164315183e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "inverse_R", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006476297043263912 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.6249348593410105e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.624933586048428e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2699590243860832e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "X_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.021607279777526855 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.766223668819293e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.7661858338397e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.7834979593753815e-10, "numeric_tolerance_ratio": 0.00037580701583076647 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "Y_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.019717823714017868 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 6.143248901935294e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 6.143239443190396e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.40099235142008e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "S_order_7", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.020352382212877274 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 7.072931475704536e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 7.072944572428241e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.3096723705530167e-10, "numeric_tolerance_ratio": 0.00013004741886984787 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "R_order_4", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.012590174563229084 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.74880874005612e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.748832386918366e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 2.3646862246096134e-10, "numeric_tolerance_ratio": 0.000235585452088286 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "H_order_2", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 6.998432908560917e-16 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_inverse", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.005898219998925924 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.7603371563600376e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.760327151918318e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.000444171950221e-10, "numeric_tolerance_ratio": 9.976902205452467e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.8386926715029404e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.8386933990987018e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 7.275957614183426e-12, "numeric_tolerance_ratio": 7.2553618663453594e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_idempotent", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.0037216152995824814 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.5795047147548757e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.579504350956995e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.637978807091713e-12, "numeric_tolerance_ratio": 3.6322416655772068e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4492263289866969e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4492271475319285e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 8.185452315956354e-12, "numeric_tolerance_ratio": 8.17360690293936e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "rotation_conjugates_translation", "group": "identities", "status": "passed", "metrics": { "decoded_relation_agreement": 1.0, "normalized_latent_relation_rmse": 0.006914579775184393 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 3.1585754186380655e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.158584877382964e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.458744898438454e-11, "numeric_tolerance_ratio": 9.428962719383602e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5493387511232868e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.54933529504342e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 3.456079866737127e-11, "numeric_tolerance_ratio": 3.447291564678985e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "translation_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5946865207515657e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.594685065560043e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.4551915228366852e-11, "numeric_tolerance_ratio": 1.4514255306884354e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5181043383781798e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5181052478728816e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 9.094947017729282e-12, "numeric_tolerance_ratio": 9.072102508792652e-06 }, "comparisons": 6 } ], "differences": [] }, { "relation": "context_flip_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 2.5509047191007994e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.5509052647976205e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 5.4569682106375694e-12, "numeric_tolerance_ratio": 5.443083420483525e-06 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 2.650252827152144e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 2.650248643476516e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 4.18367562815547e-11, "numeric_tolerance_ratio": 4.172617155200154e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_rotation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 96, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 96, "observable_mse": 2.8929631298524328e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 2.8929516702191904e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.1459633242338896e-10, "numeric_tolerance_ratio": 0.00011426576708166119 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 96, "observable_mse": 1.3614365343528334e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 96, "observable_mse": 1.3614378076454159e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.2732925824820995e-11, "numeric_tolerance_ratio": 1.2715614306756342e-05 }, "comparisons": 6 } ], "differences": [] }, { "relation": "projection_translation", "group": "noncommuting", "status": "passed", "metrics": { "oracle_distinguishing_states": 98, "distinguished_fraction": 1.0 }, "comparisons": [ { "case": "left", "status": "passed", "actual": { "count": 98, "observable_mse": 1.4222289792087395e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 1.4222278878150973e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.0913936421275139e-11, "numeric_tolerance_ratio": 1.0898436361148734e-05 }, "comparisons": 6 }, { "case": "right", "status": "passed", "actual": { "count": 98, "observable_mse": 3.5869325074600056e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "reported": { "count": 98, "observable_mse": 3.586930688470602e-05, "exact_state_accuracy": 1.0, "per_field_accuracy": { "x": 1.0, "y": 1.0, "h": 1.0 } }, "differences": [], "max_errors": { "accuracy_abs": 0.0, "numeric_abs": 1.8189894035458565e-11, "numeric_tolerance_ratio": 1.8124881342346815e-05 }, "comparisons": 6 } ], "differences": [] } ], "saved_h_count_groups": { "length": 16, "word_order": "frozen design order, zero-based", "groups": [ { "H_count": 0, "word_indices": [ 3, 16, 18, 30, 31 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 1, "word_indices": [ 0, 4, 8, 10, 13, 17, 19, 20, 28, 29 ], "count": 980, "correct_count": 980, "error_count": 0, "per_field_correct_counts": { "x": 980, "y": 980, "h": 980 } }, { "H_count": 2, "word_indices": [ 1, 2, 9, 11, 21, 22, 23 ], "count": 686, "correct_count": 686, "error_count": 0, "per_field_correct_counts": { "x": 686, "y": 686, "h": 686 } }, { "H_count": 3, "word_indices": [ 5, 6, 15, 25, 27 ], "count": 490, "correct_count": 490, "error_count": 0, "per_field_correct_counts": { "x": 490, "y": 490, "h": 490 } }, { "H_count": 4, "word_indices": [ 26 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 5, "word_indices": [ 7, 14 ], "count": 196, "correct_count": 196, "error_count": 0, "per_field_correct_counts": { "x": 196, "y": 196, "h": 196 } }, { "H_count": 7, "word_indices": [ 24 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } }, { "H_count": 8, "word_indices": [ 12 ], "count": 98, "correct_count": 98, "error_count": 0, "per_field_correct_counts": { "x": 98, "y": 98, "h": 98 } } ] }, "h_count_group_differences": [], "primary_length16": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "recorded_training": { "mode": "official", "H_edges": { "count": 84, "sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d", "total_training_edges": 580, "objective_scale": 0.14482758620689656, "scope": "H edges with both endpoints in registered training support" }, "optimizer_updates": 0, "optimizer_initial_state": { "type": null, "state": {} }, "curve": [ { "step": 0, "losses": { "prediction_mse": 2.3450114895240404e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 3.396223519303021e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 3.396223519303021e-06 }, "gradients": { "norms": { "prediction": 0.00011556209938134998, "weighted_latent": 2.6831793128829187e-18, "total": 0.00011556209938134998 }, "prediction_weighted_latent_dot": 1.8475759799887515e-23, "prediction_weighted_latent_cosine": 0.05958503346418309, "cosine_status": "defined", "semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1" }, "update_norm": 0.0 } ], "checkpoints": [ { "step": 0, "path": "checkpoints/step-0000.json", "sha256": "54d0f37cf21d2f3971d6eede53623bef7794973f5523f1f4b98bab3c2bb4a858" } ], "final_losses": { "prediction_mse": 2.3450114895240404e-05, "latent_mse": 3.151272073551623e-31, "scaled_prediction": 3.396223519303021e-06, "scaled_weighted_latent": 4.5639115510232414e-33, "total": 3.396223519303021e-06 }, "training_seconds": 4.309997893869877e-05, "evaluation_seconds": 0.6614362000254914 }, "recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed." } ], "verification": { "status": "verified", "mode": "official", "trials": 20, "optimizer_updates": 10000, "source_sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573", "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "files_verified": 324, "scope": "Pinned inputs, declared metadata migration, frozen tensors, optimizer records, outputs and resources; no retraining" }, "receipt_sha256": "cf50bd54f71a55690e3aac7e28ca33cc63f07b750964c221e2e306dab0f73125", "parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1", "run_source_sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573", "design_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df", "paired_correctness": [ { "seed": 0, "arm": "prediction_latent", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "prediction_latent", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "prediction_latent", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "prediction_latent", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "prediction_latent", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 }, "numpy": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 } }, { "seed": 0, "arm": "latent_only", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "latent_only", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "latent_only", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "latent_only", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "latent_only", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 }, "numpy": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 } }, { "seed": 0, "arm": "refit", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "refit", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "refit", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "refit", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 0, "arm": "refit", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 }, "numpy": { "count": 3136, "correct_to_correct": 2955, "correct_to_wrong": 0, "wrong_to_correct": 181, "wrong_to_wrong": 0, "net_correct_gain": 181 } }, { "seed": 1, "arm": "prediction_latent", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "prediction_latent", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "prediction_latent", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "prediction_latent", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 }, "numpy": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 } }, { "seed": 1, "arm": "prediction_latent", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 }, "numpy": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 } }, { "seed": 1, "arm": "latent_only", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "latent_only", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "latent_only", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "latent_only", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 }, "numpy": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 } }, { "seed": 1, "arm": "latent_only", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 }, "numpy": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 } }, { "seed": 1, "arm": "refit", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "refit", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "refit", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 1, "arm": "refit", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 }, "numpy": { "count": 3136, "correct_to_correct": 3059, "correct_to_wrong": 0, "wrong_to_correct": 77, "wrong_to_wrong": 0, "net_correct_gain": 77 } }, { "seed": 1, "arm": "refit", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 }, "numpy": { "count": 3136, "correct_to_correct": 3006, "correct_to_wrong": 0, "wrong_to_correct": 130, "wrong_to_wrong": 0, "net_correct_gain": 130 } }, { "seed": 2, "arm": "prediction_latent", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "prediction_latent", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "prediction_latent", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "prediction_latent", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "prediction_latent", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 }, "numpy": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 } }, { "seed": 2, "arm": "latent_only", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "latent_only", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "latent_only", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "latent_only", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "latent_only", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 }, "numpy": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 } }, { "seed": 2, "arm": "refit", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "refit", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "refit", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "refit", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 2, "arm": "refit", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 }, "numpy": { "count": 3136, "correct_to_correct": 2887, "correct_to_wrong": 0, "wrong_to_correct": 249, "wrong_to_wrong": 0, "net_correct_gain": 249 } }, { "seed": 3, "arm": "prediction_latent", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 3, "arm": "prediction_latent", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6061, "correct_to_wrong": 47, "wrong_to_correct": 67, "wrong_to_wrong": 97, "net_correct_gain": 20 }, "numpy": { "count": 6272, "correct_to_correct": 6061, "correct_to_wrong": 47, "wrong_to_correct": 67, "wrong_to_wrong": 97, "net_correct_gain": 20 } }, { "seed": 3, "arm": "prediction_latent", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2808, "correct_to_wrong": 64, "wrong_to_correct": 123, "wrong_to_wrong": 141, "net_correct_gain": 59 }, "numpy": { "count": 3136, "correct_to_correct": 2808, "correct_to_wrong": 64, "wrong_to_correct": 123, "wrong_to_wrong": 141, "net_correct_gain": 59 } }, { "seed": 3, "arm": "prediction_latent", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2097, "correct_to_wrong": 165, "wrong_to_correct": 398, "wrong_to_wrong": 476, "net_correct_gain": 233 }, "numpy": { "count": 3136, "correct_to_correct": 2097, "correct_to_wrong": 165, "wrong_to_correct": 398, "wrong_to_wrong": 476, "net_correct_gain": 233 } }, { "seed": 3, "arm": "prediction_latent", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 1615, "correct_to_wrong": 93, "wrong_to_correct": 552, "wrong_to_wrong": 876, "net_correct_gain": 459 }, "numpy": { "count": 3136, "correct_to_correct": 1615, "correct_to_wrong": 93, "wrong_to_correct": 552, "wrong_to_wrong": 876, "net_correct_gain": 459 } }, { "seed": 3, "arm": "latent_only", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 3, "arm": "latent_only", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6108, "correct_to_wrong": 0, "wrong_to_correct": 164, "wrong_to_wrong": 0, "net_correct_gain": 164 }, "numpy": { "count": 6272, "correct_to_correct": 6108, "correct_to_wrong": 0, "wrong_to_correct": 164, "wrong_to_wrong": 0, "net_correct_gain": 164 } }, { "seed": 3, "arm": "latent_only", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2872, "correct_to_wrong": 0, "wrong_to_correct": 264, "wrong_to_wrong": 0, "net_correct_gain": 264 }, "numpy": { "count": 3136, "correct_to_correct": 2872, "correct_to_wrong": 0, "wrong_to_correct": 264, "wrong_to_wrong": 0, "net_correct_gain": 264 } }, { "seed": 3, "arm": "latent_only", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2262, "correct_to_wrong": 0, "wrong_to_correct": 874, "wrong_to_wrong": 0, "net_correct_gain": 874 }, "numpy": { "count": 3136, "correct_to_correct": 2262, "correct_to_wrong": 0, "wrong_to_correct": 874, "wrong_to_wrong": 0, "net_correct_gain": 874 } }, { "seed": 3, "arm": "latent_only", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 1708, "correct_to_wrong": 0, "wrong_to_correct": 1428, "wrong_to_wrong": 0, "net_correct_gain": 1428 }, "numpy": { "count": 3136, "correct_to_correct": 1708, "correct_to_wrong": 0, "wrong_to_correct": 1428, "wrong_to_wrong": 0, "net_correct_gain": 1428 } }, { "seed": 3, "arm": "refit", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 3, "arm": "refit", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6108, "correct_to_wrong": 0, "wrong_to_correct": 164, "wrong_to_wrong": 0, "net_correct_gain": 164 }, "numpy": { "count": 6272, "correct_to_correct": 6108, "correct_to_wrong": 0, "wrong_to_correct": 164, "wrong_to_wrong": 0, "net_correct_gain": 164 } }, { "seed": 3, "arm": "refit", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2872, "correct_to_wrong": 0, "wrong_to_correct": 264, "wrong_to_wrong": 0, "net_correct_gain": 264 }, "numpy": { "count": 3136, "correct_to_correct": 2872, "correct_to_wrong": 0, "wrong_to_correct": 264, "wrong_to_wrong": 0, "net_correct_gain": 264 } }, { "seed": 3, "arm": "refit", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 2262, "correct_to_wrong": 0, "wrong_to_correct": 874, "wrong_to_wrong": 0, "net_correct_gain": 874 }, "numpy": { "count": 3136, "correct_to_correct": 2262, "correct_to_wrong": 0, "wrong_to_correct": 874, "wrong_to_wrong": 0, "net_correct_gain": 874 } }, { "seed": 3, "arm": "refit", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 1708, "correct_to_wrong": 0, "wrong_to_correct": 1428, "wrong_to_wrong": 0, "net_correct_gain": 1428 }, "numpy": { "count": 3136, "correct_to_correct": 1708, "correct_to_wrong": 0, "wrong_to_correct": 1428, "wrong_to_wrong": 0, "net_correct_gain": 1428 } }, { "seed": 4, "arm": "prediction_latent", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "prediction_latent", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "prediction_latent", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "prediction_latent", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "prediction_latent", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 }, "numpy": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 } }, { "seed": 4, "arm": "latent_only", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "latent_only", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "latent_only", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "latent_only", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "latent_only", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 }, "numpy": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 } }, { "seed": 4, "arm": "refit", "baseline": "native", "length": 1, "status": "passed", "saved": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 784, "correct_to_correct": 784, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "refit", "baseline": "native", "length": 2, "status": "passed", "saved": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 6272, "correct_to_correct": 6272, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "refit", "baseline": "native", "length": 4, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "refit", "baseline": "native", "length": 8, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 }, "numpy": { "count": 3136, "correct_to_correct": 3136, "correct_to_wrong": 0, "wrong_to_correct": 0, "wrong_to_wrong": 0, "net_correct_gain": 0 } }, { "seed": 4, "arm": "refit", "baseline": "native", "length": 16, "status": "passed", "saved": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 }, "numpy": { "count": 3136, "correct_to_correct": 3062, "correct_to_wrong": 0, "wrong_to_correct": 74, "wrong_to_wrong": 0, "net_correct_gain": 74 } } ], "H_free_isolation": [ { "seed": 0, "arm": "prediction_latent", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "prediction_latent", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "latent_only", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 0, "arm": "refit", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "prediction_latent", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "latent_only", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 1, "arm": "refit", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "prediction_latent", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "latent_only", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 2, "arm": "refit", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "prediction_latent", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "latent_only", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 3, "arm": "refit", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "prediction_latent", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "latent_only", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "reconstruction", "backend": "saved", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "reconstruction", "backend": "numpy", "status": "passed", "H_free_cases": 98, "arrays": { "latent": { "status": "passed", "compared_scalars": 980, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 1568, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "one_step", "backend": "saved", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "one_step", "backend": "numpy", "status": "passed", "H_free_cases": 686, "arrays": { "latent": { "status": "passed", "compared_scalars": 6860, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 10976, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_2", "backend": "saved", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_2", "backend": "numpy", "status": "passed", "H_free_cases": 4802, "arrays": { "latent": { "status": "passed", "compared_scalars": 48020, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 76832, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_4", "backend": "saved", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_4", "backend": "numpy", "status": "passed", "H_free_cases": 1666, "arrays": { "latent": { "status": "passed", "compared_scalars": 16660, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 26656, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_8", "backend": "saved", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_8", "backend": "numpy", "status": "passed", "H_free_cases": 784, "arrays": { "latent": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 12544, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_16", "backend": "saved", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } }, { "seed": 4, "arm": "refit", "set": "composition_16", "backend": "numpy", "status": "passed", "H_free_cases": 490, "arrays": { "latent": { "status": "passed", "compared_scalars": 4900, "differing_scalars": 0, "witnesses": [] }, "logits": { "status": "passed", "compared_scalars": 7840, "differing_scalars": 0, "witnesses": [] } } } ], "summary": { "trials_audited": 20, "trials_passed": 20, "trials_failed": 0, "distinct_saved_predictions": 331240, "categorical_disagreements": 0, "continuous_scalars_compared": 8631840, "continuous_scalars_outside_tolerance": 0, "continuous_witnesses_retained": 0, "max_continuous_absolute_error": 7.748603820800781e-07, "max_continuous_tolerance_ratio": 0.6967987208633615, "aggregate_comparison_failures": 0, "relation_failures": 0, "paired_comparison_rows": 75, "paired_comparison_failures": 0, "H_free_isolation_rows": 180, "H_free_isolation_failures": 0, "primary_length16": [ { "seed": 0, "arm": "native", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 2955, "accuracy": 0.9422831632653061 }, "numpy": { "cases": 3136, "correct": 2955, "accuracy": 0.9422831632653061 } }, "failure": null }, { "seed": 0, "arm": "prediction_latent", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 0, "arm": "latent_only", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 0, "arm": "refit", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 1, "arm": "prediction_latent", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 1, "arm": "latent_only", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 1, "arm": "refit", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 1, "arm": "native", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3006, "accuracy": 0.9585459183673469 }, "numpy": { "cases": 3136, "correct": 3006, "accuracy": 0.9585459183673469 } }, "failure": null }, { "seed": 2, "arm": "latent_only", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 2, "arm": "refit", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 2, "arm": "native", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 2887, "accuracy": 0.9205994897959183 }, "numpy": { "cases": 3136, "correct": 2887, "accuracy": 0.9205994897959183 } }, "failure": null }, { "seed": 2, "arm": "prediction_latent", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 3, "arm": "refit", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 3, "arm": "native", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 1708, "accuracy": 0.5446428571428571 }, "numpy": { "cases": 3136, "correct": 1708, "accuracy": 0.5446428571428571 } }, "failure": null }, { "seed": 3, "arm": "prediction_latent", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 2167, "accuracy": 0.6910076530612245 }, "numpy": { "cases": 3136, "correct": 2167, "accuracy": 0.6910076530612245 } }, "failure": null }, { "seed": 3, "arm": "latent_only", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 4, "arm": "native", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3062, "accuracy": 0.9764030612244898 }, "numpy": { "cases": 3136, "correct": 3062, "accuracy": 0.9764030612244898 } }, "failure": null }, { "seed": 4, "arm": "prediction_latent", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 4, "arm": "latent_only", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null }, { "seed": 4, "arm": "refit", "status": "passed", "results": { "saved": { "cases": 3136, "correct": 3136, "accuracy": 1.0 }, "numpy": { "cases": 3136, "correct": 3136, "accuracy": 1.0 } }, "failure": null } ] }, "ended_at": "2026-10-03T18:52:52.935961+00:00", "elapsed_seconds": 13.982745499990415 }