REVIEWED PUBLIC SOURCE / HISTORICAL SNAPSHOT
Independent NumPy inference replay
This is the exact UTF-8 source text at the recorded commit. The implementation repository is private. Older records may describe earlier access, hosting or release decisions; their original wording is preserved. Links and commands inside the record remain inert source text and do not require repository access to read this record.
- Original path
docs/evidence/head-fitting-replay.json- Commit
e6d9e0584066ce0fb4cfb82088c1e7490a886e7b- SHA-256
5e4cb552a118cb209e3ebe2eab1fbe50bde11b83551f3a4cb2232e342de2dcd6- Size
- 2,677,154 bytes
Download the exact original bytes · Read the source
{
"format": "renigma.head_fitting_replay",
"version": 1,
"status": "passed",
"started_at": "2026-10-03T18:52:38.953256+00:00",
"run": "G:\\RenigmaLLM_Artifacts\\head-fitting-runs\\rep-8d3baca72c53",
"source": {
"sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573",
"git_commit": "d56d2ec3308539866e48bd441ae618d12fd5319d",
"files": {
"src/renigma/__init__.py": "d0622b66407dc62610ca3eb83b960807ea26f67384f832c75f35362db73280b3",
"src/renigma/__main__.py": "935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d",
"src/renigma/cli.py": "1d8f53bfeec7598e8d35f588fa0edddd7108a65e99f63703cee334d47746d7d3",
"src/renigma/diagnostic_worker.py": "cadaa9cc4509df8bfd73212f9bd4e52000872d76cb8930deb3673c99f1542b2f",
"src/renigma/evidence.py": "06f241371dfb3820267f83446396315ad21a65019b374be594be83c9d1ae51fa",
"src/renigma/experiments.py": "cb7ec032bbb2e4e81982c7417d2e42dc34b7a26d3bbca6d60889675ec2f0f418",
"src/renigma/gpu.py": "d2adc63a6de64be7d90a8f20298ce7c40326fa5972782cfc83adbb8d8fa92f95",
"src/renigma/head_fitting.py": "e9f78f9be7b8037f0b869756473418afd8fca6a7bd16c85f701f3161d2d8704b",
"src/renigma/head_fitting_study.py": "7d5e31e262173fdeebb5eae2cd5f446ebbd34d2853d7fe0b5863addb6f2e0f69",
"src/renigma/head_intervention.py": "ef4abd910ece305f4c02dd4aa236d48fd9ba5f19d36c02d26548d6bceba1754f",
"src/renigma/head_intervention_study.py": "6f13b7b29c3386a4dee18c637eeb6807b9e62f1f78a3e792a38850137186b40a",
"src/renigma/math_checks.py": "02a036088fe46180a74cc1b1f89ffbce9c4d49b670786c6939a04c4f856bcb67",
"src/renigma/memory.py": "a42b4424e82afbbec5b2b32982e221b20f64f27f6dfdf41ddeda9d600862151a",
"src/renigma/memory_diagnostics.py": "1e45bb1c083b496d21891e8470fd1355b94d53d12010fc3fc27b4217abe9a605",
"src/renigma/memory_experiments.py": "426bd7ee4c9286119d51f29bf6120a5841031ad03af374e9a113568d8c2486f5",
"src/renigma/memory_language.py": "b2f04d3357f83e0556e3846f2e8f89e4b3e8bfc137bf7f0e3262f7e865126a6e",
"src/renigma/memory_language_v2.py": "b42047f2ee10abc39209d8b7e59e4039738d6a6c2a26d71923e74881d8a6d14b",
"src/renigma/memory_performance.py": "488dcf81a2bc1e2533de79993a26bac679c5b000e4a031281c40018a867662fd",
"src/renigma/memory_scan.py": "fe117844a25c369b484a333790b4cee99f6362d0ddf59dab7160031b362e0952",
"src/renigma/memory_scan_performance.py": "a09a63dba6860d92ed6f4bdf3f2b7a1796edb963823dc67810853158a3818c77",
"src/renigma/memory_training.py": "60fa357f0131c9af7d4a36e744f3d8f23670b9830ba8c6f99a38557e7e87f9f8",
"src/renigma/memory_worker.py": "86611aa159b6e7576995930c28e6352d7ddda3703e2bfa0c1c7706cb1c3b35df",
"src/renigma/planner.py": "18c3cae774b0b9c6aead82997a29782432ff47e58746149cefa07b7d37c7c209",
"src/renigma/representation.py": "426857a82c5e0cee3ac5505991beae26eb7e4c8b779e556248ada42d3748c6f2",
"src/renigma/representation_fixed.py": "49bc04dc438b743f0b5e680519d7e2a145e3e4ce833c916fd60e2d94e5959b0e",
"src/renigma/representation_fixed_diagnostics.py": "9e5dae675f2ef41fc59318134c7a3b2cb0cba44246b3d7defc449cf4977ee76b",
"src/renigma/representation_fixed_study.py": "4ee7ba69ecb8c0c83f8d4d4b1ed7492242be63660d09d856cc92af977436b943",
"src/renigma/representation_learning.py": "4abd7d319b643720ec36b08c9cbebb42ad10a030e786ff231912bc2f284dece3",
"src/renigma/representation_math.py": "705208f79e6ebef7fff706423bb83e39b0d835a762d0be0ea29fa6db770b619b",
"src/renigma/representation_study.py": "76b9d4b40bbf929f8d8deab9c79b57051f8552f143c55c7aaeefc23e68b12603",
"src/renigma/resources.py": "003161b14e3afdba382ee3513f0624a7f07d5be3b94b12ac23e40928242dce50",
"src/renigma/routing.py": "06e55cc7cf4bb97b66c83299fc84eea940d5713bd67c2a85ba20c155d389c4c0",
"src/renigma/server.py": "9b703b2c40faaf651a4039d770e4914cf140dfd16100b57005a0eb83583164ec",
"src/renigma/training.py": "b15ae472cae35d8f93205b69031af1105beeb7b5b2aeb459fc080f253d32ef36",
"src/renigma/web/app.js": "6a1743905a01ca5870e53c885d74327a792e76b2df30235e9c810d24b9c36baa",
"src/renigma/web/index.html": "07cae5696d05141cbee4da09cc3ad9a0d4c04080d8724d7a3a66b4a8d891f820",
"src/renigma/web/styles.css": "4c1a818b3acece76076bb1754d59339d4275d3bb0fd7b67b4d433217332a19f9",
"src/renigma/worker.py": "2b898ae8e22622d17936efe18613f470c89bfe02102f12afc0c8a85f8694cea2",
"pyproject.toml": "e65d947f319af535facb7163870751a47cf48ee200f9ddaf96ece7e2f7230033",
"uv.lock": "a23fa9ec75705d63d7d8d766489bcbcef99a6f2ed00b33d2e658a8005ce9bf86"
}
},
"script_sha256": "a54b50420af815cff97b78509a64f3064b6c8d93976c1e3005596c62ef3d697c",
"helper_script_sha256": {
"replay_representation.py": "1aee7e5d8a18b403f1dd72983c2233b2c92a016cea875b082ebc01495b797c5d",
"replay_fixed_representation.py": "96f86b9caf60761e65bd341431055b94452c53ae989c757d7292b02bf5311bf6",
"replay_head_intervention.py": "da6bdd54652127843838f02fcb12b4ff9660eb28eac26ebb31699591e7493dc9"
},
"environment": {
"python": "3.12.10",
"numpy": "2.5.3"
},
"current_source_required": true,
"tolerances": {
"continuous_atol": 1e-06,
"continuous_rtol": 0.0001,
"categorical": "exact decoded indices",
"aggregate_accuracy_atol": 1e-07
},
"scope": "Twenty final H fitting checkpoints; NumPy inference, frozen-tensor isolation, independent truth, diagnostic/H-group and paired-correctness checks. Zero GPU or optimizer calls.",
"limitations": [
"Same host, synthetic world and words; post-outcome recovery comparison, not independent-host or new-domain replication.",
"Recorded training losses/gradients and optimizer state are contextual evidence, not independently replayed optimization.",
"All continuous and categorical failures remain failures, even if task accuracy improves.",
"Relation per-example CUDA arrays were not saved; relations compare recorded aggregate metrics."
],
"trials": [
{
"seed": 0,
"arm": "native",
"status": "passed",
"files": {
"seed-0-native/result.json": "17e6bbd872fffd4273596cc9ed4ca9f4f4d40a6894150d542c660ca712aab892",
"seed-0-native/checkpoint.json": "e582fb862d817001b6c93efafcdccdc9daf7e52518b61b997694ca02a242cb57",
"inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"seed-0-native/diagnostics-arrays.npz": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 3.8743019104003906e-07,
"max_tolerance_ratio": 0.35905024250805767,
"worst_tolerance_index": [
64,
7
],
"actual_at_worst": 0.0007908046245574951,
"saved_at_worst": 0.0007904171943664551,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.15187023463659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 4.151866596657783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6229368565750986e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444363291142508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 7.444350922014564e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.0001227772822666739
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926477193832397,
"saved_at_worst": 1.2926477193832397,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4581003251396934,
"worst_tolerance_index": [
62,
8
],
"actual_at_worst": -0.0012227296829223633,
"saved_at_worst": -0.0012222155928611755,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.00018554557755123824,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04766174405813217
},
"reported": {
"count": 784,
"observable_mse": 0.00018554567941464484,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04766174405813217
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.00010000780498627342
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 0.0002031843177974224,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05125661566853523
},
"reported": {
"count": 580,
"observable_mse": 0.00020318443421274424,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05125661566853523
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011409704742430952
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.00019697788229677826,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04766174778342247
},
"reported": {
"count": 112,
"observable_mse": 0.0001969779550563544,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04766174405813217
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006460592476540826
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 6.04273664066568e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"reported": {
"count": 92,
"observable_mse": 6.0427275457186624e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-11,
"numeric_tolerance_ratio": 9.040318834088458e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.778126847464591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 98,
"observable_mse": 3.7781326682306826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987823776900768
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003417623690158127
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.11751791741699e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0049878195859491825
},
"reported": {
"count": 70,
"observable_mse": 3.1175259209703654e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987821914255619
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.001553465506768622
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.356186400400475e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 14,
"observable_mse": 4.356192584964447e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498782517388463
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004349702472956356
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987813998013735
},
"reported": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498783215880394
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.816079020500183e-08,
"numeric_tolerance_ratio": 0.012117022670509476
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7222198443487287e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297571886330843
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529756722971797
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0030440218389961885
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.335614383104257e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529757235199213
},
"reported": {
"count": 70,
"observable_mse": 3.3356191124767065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003652826317988257
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0065819373703562e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297572817653418
},
"reported": {
"count": 14,
"observable_mse": 3.0065733881201595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003957228511153945
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.370883056661114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052975681610405445
},
"reported": {
"count": 14,
"observable_mse": 6.370881601469591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297568626701832
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003044021561014145
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7371144571807235e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311058834195137
},
"reported": {
"count": 98,
"observable_mse": 3.737111546797678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311060231178999
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0009124017806934365
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.151369674014859e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311061628162861
},
"reported": {
"count": 70,
"observable_mse": 3.151365672238171e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311059299856424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0015206697269865119
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.960085814469494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311049986630678
},
"reported": {
"count": 14,
"observable_mse": 3.960098547395319e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311064422130585
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4435499906539917e-08,
"numeric_tolerance_ratio": 0.009428149153154155
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.442864105338231e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311055574566126
},
"reported": {
"count": 14,
"observable_mse": 6.442856829380617e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311062093824148
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004257874458583739
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.720241511473432e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"reported": {
"count": 98,
"observable_mse": 3.720237145898864e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681416642266
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.12783413392026e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518278449774
},
"reported": {
"count": 72,
"observable_mse": 3.127835225313902e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518744111061
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00030094078470998165
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.02617733925581e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"reported": {
"count": 14,
"observable_mse": 4.0261500544147566e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681507163908
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.917762220837176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499614402651787
},
"reported": {
"count": 12,
"observable_mse": 6.917752762092277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499619524925947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0033047741282601615
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.71916830772534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498461425304413
},
"reported": {
"count": 98,
"observable_mse": 3.719156302395277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498470738530159
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.006009125611987981
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.126173032796942e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0054947249591350555
},
"reported": {
"count": 72,
"observable_mse": 3.126166484435089e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00549473287537694
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.916241884231567e-09,
"numeric_tolerance_ratio": 0.005108988937015792
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.025795715278946e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498459096997976
},
"reported": {
"count": 14,
"observable_mse": 4.0257600630866364e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498471204191446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.210719347000122e-08,
"numeric_tolerance_ratio": 0.007811863060872042
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.919406587257981e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055208331905305386
},
"reported": {
"count": 12,
"observable_mse": 6.919389124959707e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005520846229046583
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.30385160446167e-08,
"numeric_tolerance_ratio": 0.008400647652971193
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.75562631234061e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 98,
"observable_mse": 3.755628131330013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.164217559969984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217758007347584
},
"reported": {
"count": 70,
"observable_mse": 3.164222289342433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002141989845922839
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.49347566650249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217755679041147
},
"reported": {
"count": 14,
"observable_mse": 6.493463297374547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0036719825930105815
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.001238399650901556,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405950367450714
},
"reported": {
"count": 98,
"observable_mse": 0.0012384005822241306,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405951857566833
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0010343753291123703
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.0012248106068000197,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405951857566833
},
"reported": {
"count": 84,
"observable_mse": 0.0012248115381225944,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405951857566833
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0008296999655205318
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00131993368268013,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405951857566833
},
"reported": {
"count": 14,
"observable_mse": 0.001319935079663992,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13405951857566833
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0012340917612088322
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640167688019574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707476753741503
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005929166943850281
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 2.1136473151273094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005575723946094513
},
"reported": {
"count": 72,
"observable_mse": 2.1136505893082358e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055757262744009495
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0014948300936472665
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.2392847313312814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707480479031801
},
"reported": {
"count": 14,
"observable_mse": 2.239291643491015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0017787500831550841
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.3784225049894303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441648583859205
},
"reported": {
"count": 12,
"observable_mse": 2.37841213674983e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441639736294746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.005381193482372771
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2945678234100342,
"saved_at_worst": 1.2945678234100342,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5731212112389529,
"worst_tolerance_index": [
1729,
8
],
"actual_at_worst": -0.0003994107246398926,
"saved_at_worst": -0.0004000067710876465,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.00066754239378497,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07171070575714111
},
"reported": {
"count": 6272,
"observable_mse": 0.0006675425102002919,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07171070575714111
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010913040347916929
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.0007636122754774988,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07652070373296738
},
"reported": {
"count": 4092,
"observable_mse": 0.0007636125083081424,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07652070373296738
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00021631273280615958
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.0006516751018352807,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07171070575714111
},
"reported": {
"count": 896,
"observable_mse": 0.0006516751600429416,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07171070575714111
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.464648521372365e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.00037244788836687803,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05357403680682182
},
"reported": {
"count": 1284,
"observable_mse": 0.0003724479756783694,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.05357403680682182
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.41763598862407e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.0944832563400269,
"saved_at_worst": 1.0944832563400269,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.332993507385254e-07,
"max_tolerance_ratio": 0.5058330479705814,
"worst_tolerance_index": [
1528,
4
],
"actual_at_worst": -0.0008991509675979614,
"saved_at_worst": -0.0008997023105621338,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0017343565123155713,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10217385739088058
},
"reported": {
"count": 3136,
"observable_mse": 0.001734356745146215,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10217385739088058
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00019841790113478305
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.0018767246510833502,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10587301105260849
},
"reported": {
"count": 1582,
"observable_mse": 0.0018767250003293157,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10587301105260849
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00029405914969919786
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.001719222986139357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10217385739088058
},
"reported": {
"count": 448,
"observable_mse": 0.0017192234518006444,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10217385739088058
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00039734824514860753
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.0015368456952273846,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09663684666156769
},
"reported": {
"count": 1106,
"observable_mse": 0.0015368463937193155,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09663684666156769
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0006054444231327111
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.0646353960037231,
"saved_at_worst": 1.0646353960037231,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4339908584165583,
"worst_tolerance_index": [
2639,
4
],
"actual_at_worst": 0.0016734600067138672,
"saved_at_worst": 0.001673966646194458,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.009501377120614052,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.21004949510097504
},
"reported": {
"count": 3136,
"observable_mse": 0.009501378051936626,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.21004948019981384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0006771732057861167
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.012110286392271519,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.2318454086780548
},
"reported": {
"count": 1001,
"observable_mse": 0.012110287323594093,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.23184539377689362
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0006161440977285771
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.009507632814347744,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.21004949510097504
},
"reported": {
"count": 448,
"observable_mse": 0.009507635608315468,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.21004949510097504
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0014322431379923146
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.007951690815389156,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.19597406685352325
},
"reported": {
"count": 1687,
"observable_mse": 0.007951691746711731,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.19597406685352325
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005187937648194476
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.0119585990905762,
"saved_at_worst": 1.0119585990905762,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.41252770307847064,
"worst_tolerance_index": [
1895,
0
],
"actual_at_worst": -0.000611158087849617,
"saved_at_worst": -0.0006107203662395477,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.023222893476486206,
"exact_state_accuracy": 0.9422831632653061,
"per_field_accuracy": {
"x": 0.984375,
"y": 0.9824617346938775,
"h": 0.9716198979591837
},
"normalized_latent_rmse": 0.30537670850753784
},
"reported": {
"count": 3136,
"observable_mse": 0.023222893476486206,
"exact_state_accuracy": 0.9422831535339355,
"per_field_accuracy": {
"x": 0.984375,
"y": 0.98246169090271,
"h": 0.9716199040412903
},
"normalized_latent_rmse": 0.30537667870521545
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.379116758101986e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0009449754658476739
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0301845520734787,
"exact_state_accuracy": 0.9459459459459459,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9754299754299754,
"h": 0.9705159705159705
},
"normalized_latent_rmse": 0.3464100658893585
},
"reported": {
"count": 407,
"observable_mse": 0.030184555798768997,
"exact_state_accuracy": 0.9459459185600281,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9754299521446228,
"h": 0.9705159664154053
},
"normalized_latent_rmse": 0.3464100658893585
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.7385917866773468e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0009270452850385952
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.023366834968328476,
"exact_state_accuracy": 0.9375,
"per_field_accuracy": {
"x": 0.984375,
"y": 0.984375,
"h": 0.9642857142857143
},
"normalized_latent_rmse": 0.30537667870521545
},
"reported": {
"count": 448,
"observable_mse": 0.023366834968328476,
"exact_state_accuracy": 0.9375000596046448,
"per_field_accuracy": {
"x": 0.9843750596046448,
"y": 0.9843750596046448,
"h": 0.9642857313156128
},
"normalized_latent_rmse": 0.30537667870521545
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.021952450275421143,
"exact_state_accuracy": 0.9425690486628672,
"per_field_accuracy": {
"x": 0.981587023235423,
"y": 0.9833406400701447,
"h": 0.9732573432704954
},
"normalized_latent_rmse": 0.2974605858325958
},
"reported": {
"count": 2281,
"observable_mse": 0.021952450275421143,
"exact_state_accuracy": 0.9425690770149231,
"per_field_accuracy": {
"x": 0.9815870523452759,
"y": 0.9833406209945679,
"h": 0.9732573628425598
},
"normalized_latent_rmse": 0.2974605858325958
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.9109852861530783e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128660127520561
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.758500679396093e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7585035897791386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.899485319690801e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128450579941273
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7913549022050574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7913512642262504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6242380475881334e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010601093992590904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.960090907639824e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.960088361054659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.536540241476581e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010598846711218357
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.98074698750861e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.980756446253508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.42124123167376e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03354682773351669
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00016841452452354133,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001684145099716261,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4310898925389558e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.034877024590969086
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002028434246312827,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00020284306083340198,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-10,
"numeric_tolerance_ratio": 0.00035656520299299306
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.033779799938201904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018723947869148105,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018723943503573537,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.285336176055767e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020691297948360443
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.837881275918335e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.837849989300594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.00031043304973434846
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.23123303055763245
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.009352692402899265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00935269147157669,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004812367189252784
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.23123227059841156
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.011121547780930996,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.01112155057489872,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.00132280426758386
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0048739295452833176
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.30572659347672e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.3057307771523483e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.1740513894015114e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640156774083152e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.2671108366151636e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010431777685880661
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.1628161600092426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.162837260286324e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1100277081131935e-10,
"numeric_tolerance_ratio": 0.0002099189932115251
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.72222057194449e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0873462981808747e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.857536648865789e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.8575337384827435e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8991992866106846e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7742513086413965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.774241849896498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.423179539860027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0025512666907161474,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.002551267622038722,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.000742014753139494
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.001361202565021813,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0013612019829452038,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005123371717257159
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 4.222623465466313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 4.222628194838762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.709486040680649e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 3.188768096151762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 3.1887459044810385e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.219167072325945e-10,
"numeric_tolerance_ratio": 0.0002212113205401971
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1726252927910537e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.172617289237678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.978241468780203e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.0208916945848614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.020887329010293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.343765013792109e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 979,
"error_count": 1,
"per_field_correct_counts": {
"x": 980,
"y": 979,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 489,
"error_count": 1,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 489
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 77,
"error_count": 21,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 77
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 144,
"error_count": 52,
"per_field_correct_counts": {
"x": 147,
"y": 193,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 42,
"error_count": 56,
"per_field_correct_counts": {
"x": 98,
"y": 72,
"h": 63
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 48,
"error_count": 50,
"per_field_correct_counts": {
"x": 98,
"y": 73,
"h": 66
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 2955,
"accuracy": 0.9422831632653061
},
"numpy": {
"cases": 3136,
"correct": 2955,
"accuracy": 0.9422831632653061
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0012248115381225944,
"latent_mse": 0.017971953377127647,
"scaled_prediction": 0.00017738650785759091,
"scaled_weighted_latent": 0.00026028347201645374,
"total": 0.00043766997987404466
},
"gradients": {
"norms": {
"prediction": 0.0018048041965812445,
"weighted_latent": 0.0014241763856261969,
"total": 0.0028967412654310465
},
"prediction_weighted_latent_dot": 1.5527552932326216e-06,
"prediction_weighted_latent_cosine": 0.6041004313564027,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "e582fb862d817001b6c93efafcdccdc9daf7e52518b61b997694ca02a242cb57"
}
],
"final_losses": {
"prediction_mse": 0.0012248115381225944,
"latent_mse": 0.017971953377127647,
"scaled_prediction": 0.00017738650785759091,
"scaled_weighted_latent": 0.00026028347201645374,
"total": 0.00043766997987404466
},
"training_seconds": 0.0001259999698959291,
"evaluation_seconds": 0.7381863999762572
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 0,
"arm": "prediction_latent",
"status": "passed",
"files": {
"seed-0-prediction_latent/result.json": "b96bd2283b5fe3039b095f345c3a317a648d7e275d5b5d23e25ff4556f5d74e6",
"seed-0-prediction_latent/checkpoint.json": "5841c52b66285b7a9dbfc26dcf897f5a2dfb493fc8a30c68e2de0daafc205dee",
"inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"seed-0-prediction_latent/diagnostics-arrays.npz": "4c9966db1686750b758d3fd35adc6821f0983f63b697220ab5ce91c45b8c783d"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 3.8743019104003906e-07,
"max_tolerance_ratio": 0.35905024250805767,
"worst_tolerance_index": [
64,
7
],
"actual_at_worst": 0.0007908046245574951,
"saved_at_worst": 0.0007904171943664551,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.15187023463659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 4.151866596657783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6229368565750986e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444363291142508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 7.444350922014564e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.0001227772822666739
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926477193832397,
"saved_at_worst": 1.2926477193832397,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 6.705522537231445e-07,
"max_tolerance_ratio": 0.4581003251396934,
"worst_tolerance_index": [
62,
8
],
"actual_at_worst": -0.0012227296829223633,
"saved_at_worst": -0.0012222155928611755,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 3.544173887348734e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005298294126987457
},
"reported": {
"count": 784,
"observable_mse": 3.5441717045614496e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005298294126987457
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1750784328182317e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 3.060372546315193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005271988920867443
},
"reported": {
"count": 580,
"observable_mse": 3.060372546315193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005271988920867443
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 3.9971819205675274e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529829366132617
},
"reported": {
"count": 112,
"observable_mse": 3.997175008407794e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529829366132617
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.912159733474255e-11,
"numeric_tolerance_ratio": 6.884640620045939e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 6.04273664066568e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"reported": {
"count": 92,
"observable_mse": 6.0427275457186624e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-11,
"numeric_tolerance_ratio": 9.040318834088458e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.778126847464591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 98,
"observable_mse": 3.7781326682306826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987823776900768
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003417623690158127
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.11751791741699e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0049878195859491825
},
"reported": {
"count": 70,
"observable_mse": 3.1175259209703654e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987821914255619
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.001553465506768622
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.356186400400475e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 14,
"observable_mse": 4.356192584964447e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498782517388463
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004349702472956356
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987813998013735
},
"reported": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498783215880394
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.816079020500183e-08,
"numeric_tolerance_ratio": 0.012117022670509476
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7222198443487287e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297571886330843
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529756722971797
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0030440218389961885
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.335614383104257e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529757235199213
},
"reported": {
"count": 70,
"observable_mse": 3.3356191124767065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003652826317988257
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0065819373703562e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297572817653418
},
"reported": {
"count": 14,
"observable_mse": 3.0065733881201595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003957228511153945
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.370883056661114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052975681610405445
},
"reported": {
"count": 14,
"observable_mse": 6.370881601469591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297568626701832
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003044021561014145
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7371144571807235e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311058834195137
},
"reported": {
"count": 98,
"observable_mse": 3.737111546797678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311060231178999
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0009124017806934365
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.151369674014859e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311061628162861
},
"reported": {
"count": 70,
"observable_mse": 3.151365672238171e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311059299856424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0015206697269865119
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.960085814469494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311049986630678
},
"reported": {
"count": 14,
"observable_mse": 3.960098547395319e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311064422130585
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4435499906539917e-08,
"numeric_tolerance_ratio": 0.009428149153154155
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.442864105338231e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311055574566126
},
"reported": {
"count": 14,
"observable_mse": 6.442856829380617e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311062093824148
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004257874458583739
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.720241511473432e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"reported": {
"count": 98,
"observable_mse": 3.720237145898864e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681416642266
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.12783413392026e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518278449774
},
"reported": {
"count": 72,
"observable_mse": 3.127835225313902e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518744111061
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00030094078470998165
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.02617733925581e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"reported": {
"count": 14,
"observable_mse": 4.0261500544147566e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681507163908
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.917762220837176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499614402651787
},
"reported": {
"count": 12,
"observable_mse": 6.917752762092277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499619524925947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0033047741282601615
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.71916830772534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498461425304413
},
"reported": {
"count": 98,
"observable_mse": 3.719156302395277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498470738530159
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.006009125611987981
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.126173032796942e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0054947249591350555
},
"reported": {
"count": 72,
"observable_mse": 3.126166484435089e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00549473287537694
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.916241884231567e-09,
"numeric_tolerance_ratio": 0.005108988937015792
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.025795715278946e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498459096997976
},
"reported": {
"count": 14,
"observable_mse": 4.0257600630866364e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498471204191446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.210719347000122e-08,
"numeric_tolerance_ratio": 0.007811863060872042
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.919406587257981e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055208331905305386
},
"reported": {
"count": 12,
"observable_mse": 6.919389124959707e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005520846229046583
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.30385160446167e-08,
"numeric_tolerance_ratio": 0.008400647652971193
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.75562631234061e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 98,
"observable_mse": 3.755628131330013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.164217559969984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217758007347584
},
"reported": {
"count": 70,
"observable_mse": 3.164222289342433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002141989845922839
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.49347566650249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217755679041147
},
"reported": {
"count": 14,
"observable_mse": 6.493463297374547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0036719825930105815
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.756877777050249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004836500622332096
},
"reported": {
"count": 98,
"observable_mse": 3.756875958060846e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0048364996910095215
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006277239200697941
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 3.3182703191414475e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0048364996910095215
},
"reported": {
"count": 84,
"observable_mse": 3.318267044960521e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004836499225348234
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003138619698858303
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.388524343492463e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004836504813283682
},
"reported": {
"count": 14,
"observable_mse": 6.388530164258555e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004836502484977245
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.001569309504646551
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640167688019574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707476753741503
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005929166943850281
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 2.1136473151273094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005575723946094513
},
"reported": {
"count": 72,
"observable_mse": 2.1136505893082358e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055757262744009495
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0014948300936472665
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.2392847313312814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707480479031801
},
"reported": {
"count": 14,
"observable_mse": 2.239291643491015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0017787500831550841
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.3784225049894303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441648583859205
},
"reported": {
"count": 12,
"observable_mse": 2.37841213674983e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441639736294746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.005381193482372771
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2945678234100342,
"saved_at_worst": 1.2945678234100342,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5731212112389529,
"worst_tolerance_index": [
1729,
8
],
"actual_at_worst": -0.0003994107246398926,
"saved_at_worst": -0.0004000067710876465,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 3.810602720477618e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010186481289565563
},
"reported": {
"count": 6272,
"observable_mse": 3.810604175669141e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010186481289565563
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4496674141350507e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 3.450327494647354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010137348435819149
},
"reported": {
"count": 4092,
"observable_mse": 3.450328949838877e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010137349367141724
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00046248518493457986
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 4.079767313669436e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010186480358242989
},
"reported": {
"count": 896,
"observable_mse": 4.07976949645672e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010186480358242989
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1739181990985534e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 4.770941086462699e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010341506451368332
},
"reported": {
"count": 1284,
"observable_mse": 4.770942541654222e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.010341506451368332
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4482818533303257e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.276785135269165,
"saved_at_worst": 1.276785135269165,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.332993507385254e-07,
"max_tolerance_ratio": 0.5058330479705814,
"worst_tolerance_index": [
1528,
4
],
"actual_at_worst": -0.0008991509675979614,
"saved_at_worst": -0.0008997023105621338,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 9.610591223463416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019914837554097176
},
"reported": {
"count": 3136,
"observable_mse": 9.610602864995599e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019914837554097176
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011530715059507135
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 9.450990910409018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019731320440769196
},
"reported": {
"count": 1582,
"observable_mse": 9.451002551941201e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019731322303414345
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006264925354554616
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 9.646607941249385e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019914837554097176
},
"reported": {
"count": 448,
"observable_mse": 9.646618855185807e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.019914837554097176
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-10,
"numeric_tolerance_ratio": 0.00010809659753677173
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 9.824293374549598e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020174432545900345
},
"reported": {
"count": 1106,
"observable_mse": 9.824305743677542e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020174432545900345
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.00012248792065866024
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.279247760772705,
"saved_at_worst": 1.279247760772705,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4339908584165583,
"worst_tolerance_index": [
2639,
4
],
"actual_at_worst": 0.0016734600067138672,
"saved_at_worst": 0.001673966646194458,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0004068507405463606,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.038872551172971725
},
"reported": {
"count": 3136,
"observable_mse": 0.00040685111889615655,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03887254744768143
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0007622459832792379
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.0004357768630143255,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03872304782271385
},
"reported": {
"count": 1001,
"observable_mse": 0.00043577715405263007,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03872304782271385
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002788851278357503
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00040604922105558217,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.038872551172971725
},
"reported": {
"count": 448,
"observable_mse": 0.0004060495411977172,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03887254744768143
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0007622459832792379
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.0003898999420925975,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03896098956465721
},
"reported": {
"count": 1687,
"observable_mse": 0.0003899003495462239,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03896098956465721
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.0745362639427185e-10,
"numeric_tolerance_ratio": 0.0003921631706622357
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2058404684066772,
"saved_at_worst": 1.2058404684066772,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.811452865600586e-07,
"max_tolerance_ratio": 0.49983340846515517,
"worst_tolerance_index": [
2358,
8
],
"actual_at_worst": 0.0001366734504699707,
"saved_at_worst": 0.00013616681098937988,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0017277415608987212,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0770382285118103
},
"reported": {
"count": 3136,
"observable_mse": 0.0017277424922212958,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0770382210612297
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0008560125087669803
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0017671252135187387,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07580389082431793
},
"reported": {
"count": 407,
"observable_mse": 0.0017671257955953479,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07580388337373734
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0008683267358041612
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0017325704684481025,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0770382210612297
},
"reported": {
"count": 448,
"observable_mse": 0.0017325716326013207,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0770382210612297
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0009922404522419563
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.001719765947200358,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07725640386343002
},
"reported": {
"count": 2281,
"observable_mse": 0.0017197668785229325,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07725639641284943
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.000853872140406964
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128660127520561
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.758500679396093e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7585035897791386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.899485319690801e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128450579941273
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7913549022050574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7913512642262504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6242380475881334e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010601093992590904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.960090907639824e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.960088361054659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.536540241476581e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010598846711218357
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.98074698750861e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.980756446253508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.42124123167376e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03354682773351669
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00016841452452354133,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001684145099716261,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4310898925389558e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.034877024590969086
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002028434246312827,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00020284306083340198,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-10,
"numeric_tolerance_ratio": 0.00035656520299299306
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.033779799938201904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018723947869148105,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018723943503573537,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.285336176055767e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020691297948360443
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.837881275918335e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.837849989300594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.00031043304973434846
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.009375471621751785
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.292930316296406e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.292925586923957e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.709156391254385e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01870586909353733
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.81131854839623e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.81130326888524e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.5279510989785194e-10,
"numeric_tolerance_ratio": 0.0001514605451016903
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0048739295452833176
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.30572659347672e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.3057307771523483e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.1740513894015114e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640156774083152e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.2671108366151636e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010431777685880661
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.1628161600092426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.162837260286324e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1100277081131935e-10,
"numeric_tolerance_ratio": 0.0002099189932115251
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.72222057194449e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0873462981808747e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.857536648865789e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.8575337384827435e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8991992866106846e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7742513086413965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.774241849896498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.423179539860027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.8412308640545234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.8412283174693584e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5368405810872216e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.37335365859326e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.3733540223911405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6221379156685686e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 4.222623465466313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 4.222628194838762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.709486040680649e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 3.188768096151762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 3.1887459044810385e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.219167072325945e-10,
"numeric_tolerance_ratio": 0.0002212113205401971
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1726252927910537e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.172617289237678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.978241468780203e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.0208916945848614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.020887329010293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.343765013792109e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0012248115381225944,
"latent_mse": 0.017971953377127647,
"scaled_prediction": 0.00017738650785759091,
"scaled_weighted_latent": 0.00026028347201645374,
"total": 0.00043766997987404466
},
"gradients": {
"norms": {
"prediction": 0.0018048041965812445,
"weighted_latent": 0.0014241763856261969,
"total": 0.0028967412654310465
},
"prediction_weighted_latent_dot": 1.5527552932326216e-06,
"prediction_weighted_latent_cosine": 0.6041004313564027,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0012890921207144856,
"latent_mse": 0.016826288774609566,
"scaled_prediction": 0.00018669611017685384,
"scaled_weighted_latent": 0.0002436910872347653,
"total": 0.00043038721196353436
},
"gradients": {
"norms": {
"prediction": 0.0021970062516629696,
"weighted_latent": 0.0013744073221459985,
"total": 0.003139321692287922
},
"prediction_weighted_latent_dot": 1.5697547723902971e-06,
"prediction_weighted_latent_cosine": 0.5198583920528568,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.030365290120244026
},
{
"step": 100,
"losses": {
"prediction_mse": 3.1077976018423215e-05,
"latent_mse": 0.00010193680645897985,
"scaled_prediction": 4.500948307395447e-06,
"scaled_weighted_latent": 1.4763262470296468e-06,
"total": 5.9772746681119315e-06
},
"gradients": {
"norms": {
"prediction": 7.528421701863408e-05,
"weighted_latent": 0.00011191774683538824,
"total": 0.0001511855807621032
},
"prediction_weighted_latent_dot": 2.3318922348636306e-09,
"prediction_weighted_latent_cosine": 0.27676143903546707,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0008034161874093115
},
{
"step": 200,
"losses": {
"prediction_mse": 3.287775689386763e-05,
"latent_mse": 2.3269645680557005e-05,
"scaled_prediction": 4.761606305692112e-06,
"scaled_weighted_latent": 3.3700865742503083e-07,
"total": 5.098615019960562e-06
},
"gradients": {
"norms": {
"prediction": 5.176300328457728e-05,
"weighted_latent": 5.117325781611726e-05,
"total": 6.790669431211427e-07
},
"prediction_weighted_latent_dot": -2.64882471512351e-09,
"prediction_weighted_latent_cosine": -0.9999785580085703,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.1257843551866245e-06
},
{
"step": 300,
"losses": {
"prediction_mse": 3.285860293544829e-05,
"latent_mse": 2.3463931938749738e-05,
"scaled_prediction": 4.758832346851705e-06,
"scaled_weighted_latent": 3.398224635020597e-07,
"total": 5.098654582980089e-06
},
"gradients": {
"norms": {
"prediction": 5.168217103346251e-05,
"weighted_latent": 5.139409040566534e-05,
"total": 3.7041045288788155e-06
},
"prediction_weighted_latent_dot": -2.649339858606936e-09,
"prediction_weighted_latent_cosine": -0.9974330173843801,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.452952973428182e-05
},
{
"step": 400,
"losses": {
"prediction_mse": 3.285737329861149e-05,
"latent_mse": 2.34740182349924e-05,
"scaled_prediction": 4.758654085890157e-06,
"scaled_weighted_latent": 3.3996855108853197e-07,
"total": 5.098622750665527e-06
},
"gradients": {
"norms": {
"prediction": 5.1515187806217e-05,
"weighted_latent": 5.141399378771894e-05,
"total": 1.8833698050002567e-06
},
"prediction_weighted_latent_dot": -2.6468331970619374e-09,
"prediction_weighted_latent_cosine": -0.9993323462402814,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.012595672975294e-05
},
{
"step": 500,
"losses": {
"prediction_mse": 3.286363062215969e-05,
"latent_mse": 2.3442897145287134e-05,
"scaled_prediction": 4.759560397360474e-06,
"scaled_weighted_latent": 3.395178396203846e-07,
"total": 5.099078407511115e-06
},
"gradients": {
"norms": {
"prediction": 5.302550198393874e-05,
"weighted_latent": 5.134254024596885e-05,
"total": 1.133940622821683e-05
},
"prediction_weighted_latent_dot": -2.6595889934810657e-09,
"prediction_weighted_latent_cosine": -0.9769051209153155,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00024398861569352448
},
{
"step": 600,
"losses": {
"prediction_mse": 3.291038956376724e-05,
"latent_mse": 2.338582999072969e-05,
"scaled_prediction": 4.766332494909875e-06,
"scaled_weighted_latent": 3.3869133631014847e-07,
"total": 5.105023774376605e-06
},
"gradients": {
"norms": {
"prediction": 6.272921746131033e-05,
"weighted_latent": 5.1213362894486636e-05,
"total": 3.5458531783660874e-05
},
"prediction_weighted_latent_dot": -2.6502278149820313e-09,
"prediction_weighted_latent_cosine": -0.8249545903674677,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0006560756010003388
},
{
"step": 700,
"losses": {
"prediction_mse": 3.305563950561918e-05,
"latent_mse": 2.3388383851852268e-05,
"scaled_prediction": 4.787368652614532e-06,
"scaled_weighted_latent": 3.3872831295411743e-07,
"total": 5.1260967666166835e-06
},
"gradients": {
"norms": {
"prediction": 8.387726120417938e-05,
"weighted_latent": 5.115637759445235e-05,
"total": 6.733044574502856e-05
},
"prediction_weighted_latent_dot": -2.559490397402442e-09,
"prediction_weighted_latent_cosine": -0.5964986690100962,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0019195745699107647
},
{
"step": 800,
"losses": {
"prediction_mse": 3.2857918995432556e-05,
"latent_mse": 2.3521432012785226e-05,
"scaled_prediction": 4.758733211929211e-06,
"scaled_weighted_latent": 3.4065521958837053e-07,
"total": 5.09938854520442e-06
},
"gradients": {
"norms": {
"prediction": 5.2239465730963275e-05,
"weighted_latent": 5.15259598614648e-05,
"total": 1.2796896044164896e-05
},
"prediction_weighted_latent_dot": -2.610063054575562e-09,
"prediction_weighted_latent_cosine": -0.9696749618716722,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.000438886258052662
},
{
"step": 900,
"losses": {
"prediction_mse": 3.3233754948014393e-05,
"latent_mse": 2.3964656065800227e-05,
"scaled_prediction": 4.813164650840918e-06,
"scaled_weighted_latent": 3.4707434792835556e-07,
"total": 5.160239197721239e-06
},
"gradients": {
"norms": {
"prediction": 0.00012345743016339839,
"weighted_latent": 5.250451431493275e-05,
"total": 0.00012060565495630726
},
"prediction_weighted_latent_dot": -1.726368381582688e-09,
"prediction_weighted_latent_cosine": -0.26632969713607313,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0023534814827144146
},
{
"step": 1000,
"losses": {
"prediction_mse": 3.318267044960521e-05,
"latent_mse": 2.3391723516397178e-05,
"scaled_prediction": 4.805766366189346e-06,
"scaled_weighted_latent": 3.38776686703568e-07,
"total": 5.144543138158042e-06
},
"gradients": {
"norms": {
"prediction": 0.0001157012811745517,
"weighted_latent": 5.10505742568057e-05,
"total": 0.0001058107809512876
},
"prediction_weighted_latent_dot": -2.3985131658577075e-09,
"prediction_weighted_latent_cosine": -0.40607224546774595,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.001952397171407938
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "d49500456491e7766dc4885ab7c8a8b507ac8c95ca6f98d055d738e160c882af"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "7f5b766ed5121f19061035d8f83fd59f7b0f5632a11831bbf58a0ca2e3ef6e78"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "48dcbc552dba84b083d15627725a42753ad419ceae8d9f94dd08233d81d77f76"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "fd3c2fa5de0382152fcd0585bc269f8d60009f8fd0fd0e511b8107f09d9bdd78"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "d9e7afc97416c000c1dbe7ffd2789cdd9bf0e9762838172802cd6fcb5d84af3c"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "9212ac911a1e8a2941591140ad59df931d14a110fd616647edbc1d802e1be7c3"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "f645318aeed46a299612a4e849ecae3ba417d5bb05a118a6fda9fdfce1f0f131"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "d8ce3b1b087be62d94cc2c4188c1dc719b13cfbd1dfe9e1a1f6067d3b9a0257c"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "dbc77a4be9a4ce76a0d5b0519d01e22dfc900db42d6feae64113d06899cf38ae"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "bcdd748c39e446890935b2e68525714bb29fabfd5125318e83f42d8d1e373ad4"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "cb11b53b84855d552d86b9283b6b27b796cebf7d61ac27d0a15396ed9bfe5b52"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "5841c52b66285b7a9dbfc26dcf897f5a2dfb493fc8a30c68e2de0daafc205dee"
}
],
"final_losses": {
"prediction_mse": 3.318267044960521e-05,
"latent_mse": 2.3391723516397178e-05,
"scaled_prediction": 4.805766366189346e-06,
"scaled_weighted_latent": 3.38776686703568e-07,
"total": 5.144543138158042e-06
},
"training_seconds": 3.7349936999962665,
"evaluation_seconds": 0.5165338999941014
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 0,
"arm": "latent_only",
"status": "passed",
"files": {
"seed-0-latent_only/result.json": "3f52ebb15056db2e1e07bf859380250e0b771714ce9588fc4f64d80b36c7bb9b",
"seed-0-latent_only/checkpoint.json": "d72fb76a891400e98d3b3de0e8d37bce8f8b9352e6c9f04e76f7a9779a002ce8",
"inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"seed-0-latent_only/diagnostics-arrays.npz": "3da23297be049aeac4fd20279ba3652e82a286efbfe76a421e116b7d4b52ce34"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 3.8743019104003906e-07,
"max_tolerance_ratio": 0.35905024250805767,
"worst_tolerance_index": [
64,
7
],
"actual_at_worst": 0.0007908046245574951,
"saved_at_worst": 0.0007904171943664551,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.15187023463659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 4.151866596657783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6229368565750986e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444363291142508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 7.444350922014564e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.0001227772822666739
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926477193832397,
"saved_at_worst": 1.2926477193832397,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 7.078051567077637e-07,
"max_tolerance_ratio": 0.4581003251396934,
"worst_tolerance_index": [
62,
8
],
"actual_at_worst": -0.0012227296829223633,
"saved_at_worst": -0.0012222155928611755,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 3.652337909443304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014773458242416
},
"reported": {
"count": 784,
"observable_mse": 3.65233572665602e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014773458242416
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1748440237272644e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 3.181095598847605e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004940252751111984
},
"reported": {
"count": 580,
"observable_mse": 3.1810952350497246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004940252751111984
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.626442747347944e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 4.129156150156632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014772526919842
},
"reported": {
"count": 112,
"observable_mse": 4.129150329390541e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014772526919842
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.79683010839524e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 6.04273664066568e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"reported": {
"count": 92,
"observable_mse": 6.0427275457186624e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-11,
"numeric_tolerance_ratio": 9.040318834088458e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.778126847464591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 98,
"observable_mse": 3.7781326682306826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987823776900768
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003417623690158127
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.11751791741699e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0049878195859491825
},
"reported": {
"count": 70,
"observable_mse": 3.1175259209703654e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987821914255619
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.001553465506768622
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.356186400400475e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 14,
"observable_mse": 4.356192584964447e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498782517388463
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004349702472956356
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987813998013735
},
"reported": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498783215880394
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.816079020500183e-08,
"numeric_tolerance_ratio": 0.012117022670509476
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7222198443487287e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297571886330843
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529756722971797
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0030440218389961885
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.335614383104257e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529757235199213
},
"reported": {
"count": 70,
"observable_mse": 3.3356191124767065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003652826317988257
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0065819373703562e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297572817653418
},
"reported": {
"count": 14,
"observable_mse": 3.0065733881201595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003957228511153945
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.370883056661114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052975681610405445
},
"reported": {
"count": 14,
"observable_mse": 6.370881601469591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297568626701832
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003044021561014145
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7371144571807235e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311058834195137
},
"reported": {
"count": 98,
"observable_mse": 3.737111546797678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311060231178999
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0009124017806934365
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.151369674014859e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311061628162861
},
"reported": {
"count": 70,
"observable_mse": 3.151365672238171e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311059299856424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0015206697269865119
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.960085814469494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311049986630678
},
"reported": {
"count": 14,
"observable_mse": 3.960098547395319e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311064422130585
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4435499906539917e-08,
"numeric_tolerance_ratio": 0.009428149153154155
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.442864105338231e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311055574566126
},
"reported": {
"count": 14,
"observable_mse": 6.442856829380617e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311062093824148
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004257874458583739
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.720241511473432e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"reported": {
"count": 98,
"observable_mse": 3.720237145898864e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681416642266
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.12783413392026e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518278449774
},
"reported": {
"count": 72,
"observable_mse": 3.127835225313902e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518744111061
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00030094078470998165
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.02617733925581e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"reported": {
"count": 14,
"observable_mse": 4.0261500544147566e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681507163908
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.917762220837176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499614402651787
},
"reported": {
"count": 12,
"observable_mse": 6.917752762092277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499619524925947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0033047741282601615
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.71916830772534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498461425304413
},
"reported": {
"count": 98,
"observable_mse": 3.719156302395277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498470738530159
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.006009125611987981
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.126173032796942e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0054947249591350555
},
"reported": {
"count": 72,
"observable_mse": 3.126166484435089e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00549473287537694
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.916241884231567e-09,
"numeric_tolerance_ratio": 0.005108988937015792
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.025795715278946e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498459096997976
},
"reported": {
"count": 14,
"observable_mse": 4.0257600630866364e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498471204191446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.210719347000122e-08,
"numeric_tolerance_ratio": 0.007811863060872042
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.919406587257981e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055208331905305386
},
"reported": {
"count": 12,
"observable_mse": 6.919389124959707e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005520846229046583
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.30385160446167e-08,
"numeric_tolerance_ratio": 0.008400647652971193
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.75562631234061e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 98,
"observable_mse": 3.755628131330013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.164217559969984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217758007347584
},
"reported": {
"count": 70,
"observable_mse": 3.164222289342433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002141989845922839
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.49347566650249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217755679041147
},
"reported": {
"count": 14,
"observable_mse": 6.493463297374547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0036719825930105815
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622189226211049e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.541295091556094e-07
},
"reported": {
"count": 98,
"observable_mse": 4.6221881348174065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5176362921920372e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3658799364056904e-09,
"numeric_tolerance_ratio": 0.002365844031498052
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.1518342186464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5466694947008364e-07
},
"reported": {
"count": 84,
"observable_mse": 4.151833854848519e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5143542952955613e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2315199405275052e-09,
"numeric_tolerance_ratio": 0.0032314710046075458
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444318907801062e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5086457949564647e-07
},
"reported": {
"count": 14,
"observable_mse": 7.444314542226493e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5371813333331374e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.8535538376672775e-09,
"numeric_tolerance_ratio": 0.002853509974044612
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640167688019574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707476753741503
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005929166943850281
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 2.1136473151273094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005575723946094513
},
"reported": {
"count": 72,
"observable_mse": 2.1136505893082358e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055757262744009495
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0014948300936472665
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.2392847313312814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707480479031801
},
"reported": {
"count": 14,
"observable_mse": 2.239291643491015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0017787500831550841
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.3784225049894303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441648583859205
},
"reported": {
"count": 12,
"observable_mse": 2.37841213674983e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441639736294746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.005381193482372771
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2945678234100342,
"saved_at_worst": 1.2945678234100342,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5731212112389529,
"worst_tolerance_index": [
1729,
8
],
"actual_at_worst": -0.0003994107246398926,
"saved_at_worst": -0.0004000067710876465,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 3.762717096833512e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009358827956020832
},
"reported": {
"count": 6272,
"observable_mse": 3.7627178244292736e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009358827024698257
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00048108419659274
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 3.382581053301692e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009196789935231209
},
"reported": {
"count": 4092,
"observable_mse": 3.382582144695334e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009196789935231209
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0877143589583724e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 4.0451624954584986e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009358827024698257
},
"reported": {
"count": 896,
"observable_mse": 4.045164678245783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009358827024698257
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1739931240588358e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 4.777082722284831e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009857474826276302
},
"reported": {
"count": 1284,
"observable_mse": 4.777083086082712e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.009857474826276302
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6206825059325476e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2796679735183716,
"saved_at_worst": 1.2796679735183716,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.332993507385254e-07,
"max_tolerance_ratio": 0.5058330479705814,
"worst_tolerance_index": [
1528,
4
],
"actual_at_worst": -0.0008991509675979614,
"saved_at_worst": -0.0008997023105621338,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 8.320776396431029e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01805855706334114
},
"reported": {
"count": 3136,
"observable_mse": 8.320785127580166e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01805855706334114
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.65909863785599e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 8.007911674212664e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01770593412220478
},
"reported": {
"count": 1582,
"observable_mse": 8.007918222574517e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01770593412220478
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.548361852765083e-11,
"numeric_tolerance_ratio": 6.496339695735568e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 8.366630208911374e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01805855520069599
},
"reported": {
"count": 448,
"observable_mse": 8.366639667656273e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01805855520069599
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.380263612802507e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 8.749716653255746e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0185512974858284
},
"reported": {
"count": 1106,
"observable_mse": 8.74972902238369e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0185512974858284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.00012261840165349238
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2795177698135376,
"saved_at_worst": 1.2795177698135376,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4339908584165583,
"worst_tolerance_index": [
2639,
4
],
"actual_at_worst": 0.0016734600067138672,
"saved_at_worst": 0.001673966646194458,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00030474868253804743,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.033686716109514236
},
"reported": {
"count": 3136,
"observable_mse": 0.000304748973576352,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.033686716109514236
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002824312414728621
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00030883034924045205,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03248004615306854
},
"reported": {
"count": 1001,
"observable_mse": 0.0003088305820710957,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03248004615306854
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00022585553404942346
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.000303991895634681,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.033686719834804535
},
"reported": {
"count": 448,
"observable_mse": 0.0003039921575691551,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.033686719834804535
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.6193447411060333e-10,
"numeric_tolerance_ratio": 0.00025420678714141903
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00030252774013206363,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.034382693469524384
},
"reported": {
"count": 1687,
"observable_mse": 0.0003025280893780291,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.034382693469524384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.0003389905491651891
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2083498239517212,
"saved_at_worst": 1.2083498239517212,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.364418029785156e-07,
"max_tolerance_ratio": 0.40262168358783795,
"worst_tolerance_index": [
2786,
5
],
"actual_at_worst": 0.0007334277033805847,
"saved_at_worst": 0.0007329955697059631,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0013606043066829443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06840331107378006
},
"reported": {
"count": 3136,
"observable_mse": 0.0013606048887595534,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06840331852436066
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0009502889338298688
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0012911871308460832,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06452718377113342
},
"reported": {
"count": 407,
"observable_mse": 0.0012911877129226923,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06452718377113342
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005155140663089774
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.001364032388664782,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06840331107378006
},
"reported": {
"count": 448,
"observable_mse": 0.001364033087156713,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06840331107378006
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0006146514407380804
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0013723173178732395,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06907206773757935
},
"reported": {
"count": 2281,
"observable_mse": 0.0013723178999498487,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06907206773757935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005118363857356124
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128660127520561
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.758500679396093e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7585035897791386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.899485319690801e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128450579941273
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7913549022050574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7913512642262504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6242380475881334e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010601093992590904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.960090907639824e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.960088361054659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.536540241476581e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010598846711218357
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.98074698750861e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.980756446253508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.42124123167376e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03354682773351669
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00016841452452354133,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001684145099716261,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4310898925389558e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.034877024590969086
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002028434246312827,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00020284306083340198,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-10,
"numeric_tolerance_ratio": 0.00035656520299299306
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.033779799938201904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018723947869148105,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018723943503573537,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.285336176055767e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020691297948360443
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.837881275918335e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.837849989300594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.00031043304973434846
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 2.529685048102692e-07
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.6221557568060234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622161577572115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.793985354858499e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00969147402793169
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.399236058816314e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.3992276914650574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.36735125631094e-11,
"numeric_tolerance_ratio": 8.330702598749163e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0048739295452833176
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.30572659347672e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.3057307771523483e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.1740513894015114e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640156774083152e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.2671108366151636e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010431777685880661
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.1628161600092426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.162837260286324e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1100277081131935e-10,
"numeric_tolerance_ratio": 0.0002099189932115251
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.72222057194449e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0873462981808747e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.857536648865789e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.8575337384827435e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8991992866106846e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7742513086413965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.774241849896498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.423179539860027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.755614307010546e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7556128518190235e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4497468349913079e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.206630910630338e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.2066290916409343e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8113696433085992e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 4.222623465466313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 4.222628194838762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.709486040680649e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 3.188768096151762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 3.1887459044810385e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.219167072325945e-10,
"numeric_tolerance_ratio": 0.0002212113205401971
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1726252927910537e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.172617289237678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.978241468780203e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.0208916945848614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.020887329010293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.343765013792109e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0012248115381225944,
"latent_mse": 0.017971953377127647,
"scaled_prediction": 0.00017738650785759091,
"scaled_weighted_latent": 0.00026028347201645374,
"total": 0.00026028347201645374
},
"gradients": {
"norms": {
"prediction": 0.0018048041965812445,
"weighted_latent": 0.0014241763856261969,
"total": 0.0014241763856261969
},
"prediction_weighted_latent_dot": 1.5527552932326216e-06,
"prediction_weighted_latent_cosine": 0.6041004313564027,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0013200333341956139,
"latent_mse": 0.01672578603029251,
"scaled_prediction": 0.00019117724150419235,
"scaled_weighted_latent": 0.0002422355319140479,
"total": 0.0002422355319140479
},
"gradients": {
"norms": {
"prediction": 0.0024946476332843304,
"weighted_latent": 0.0013707657344639301,
"total": 0.0013707657344639301
},
"prediction_weighted_latent_dot": 1.628814516152488e-06,
"prediction_weighted_latent_cosine": 0.47632039876463655,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.03142811357975006
},
{
"step": 100,
"losses": {
"prediction_mse": 5.2756164222955704e-05,
"latent_mse": 5.09874735143967e-05,
"scaled_prediction": 7.640547664777841e-06,
"scaled_weighted_latent": 7.384392688436492e-07,
"total": 7.384392688436492e-07
},
"gradients": {
"norms": {
"prediction": 0.0006072220276109874,
"weighted_latent": 8.430238085566089e-05,
"total": 8.430238085566089e-05
},
"prediction_weighted_latent_dot": 2.7542473191033423e-08,
"prediction_weighted_latent_cosine": 0.5380412557577855,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0007330543594434857
},
{
"step": 200,
"losses": {
"prediction_mse": 4.1588969907024875e-05,
"latent_mse": 7.497436005365898e-10,
"scaled_prediction": 6.023230071150465e-06,
"scaled_weighted_latent": 1.0858355881104842e-11,
"total": 1.0858355881104842e-11
},
"gradients": {
"norms": {
"prediction": 0.00019014689314644784,
"weighted_latent": 3.234289351894404e-07,
"total": 3.234289351894404e-07
},
"prediction_weighted_latent_dot": 4.905533965149367e-11,
"prediction_weighted_latent_cosine": 0.7976606761791432,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.771597794686386e-06
},
{
"step": 300,
"losses": {
"prediction_mse": 4.151830944465473e-05,
"latent_mse": 2.4699960162583544e-14,
"scaled_prediction": 6.012996891513467e-06,
"scaled_weighted_latent": 3.577235600260868e-16,
"total": 3.577235600260868e-16
},
"gradients": {
"norms": {
"prediction": 0.00018826796440407634,
"weighted_latent": 1.5522670926415572e-09,
"total": 1.5522670926415572e-09
},
"prediction_weighted_latent_dot": -2.2286283875207896e-13,
"prediction_weighted_latent_cosine": -0.7625964521082172,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.203239037295134e-07
},
{
"step": 400,
"losses": {
"prediction_mse": 4.151829853071831e-05,
"latent_mse": 2.7135230707916493e-14,
"scaled_prediction": 6.012995072524063e-06,
"scaled_weighted_latent": 3.92993006098131e-16,
"total": 3.92993006098131e-16
},
"gradients": {
"norms": {
"prediction": 0.00018826920131687075,
"weighted_latent": 1.6459560381321126e-09,
"total": 1.6459560381321126e-09
},
"prediction_weighted_latent_dot": -2.2288270675688976e-13,
"prediction_weighted_latent_cosine": -0.7192483290881527,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.580078929919182e-07
},
{
"step": 500,
"losses": {
"prediction_mse": 4.151827670284547e-05,
"latent_mse": 2.31092221232446e-14,
"scaled_prediction": 6.012991889292607e-06,
"scaled_weighted_latent": 3.346852961821743e-16,
"total": 3.346852961821743e-16
},
"gradients": {
"norms": {
"prediction": 0.0001882680953713134,
"weighted_latent": 1.5939622954874721e-09,
"total": 1.5939622954874721e-09
},
"prediction_weighted_latent_dot": -2.3982706554188937e-13,
"prediction_weighted_latent_cosine": -0.7991778167082637,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.1843847147474662e-07
},
{
"step": 600,
"losses": {
"prediction_mse": 4.151826578890905e-05,
"latent_mse": 2.3013358322406148e-14,
"scaled_prediction": 6.0129905250505544e-06,
"scaled_weighted_latent": 3.332969297722857e-16,
"total": 3.332969297722857e-16
},
"gradients": {
"norms": {
"prediction": 0.0001882680953713134,
"weighted_latent": 1.5846957079901358e-09,
"total": 1.5846957079901358e-09
},
"prediction_weighted_latent_dot": -2.1274693432719177e-13,
"prediction_weighted_latent_cosine": -0.7130840130378252,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.8792765388297994e-07
},
{
"step": 700,
"losses": {
"prediction_mse": 4.151829853071831e-05,
"latent_mse": 1.8244039363003034e-14,
"scaled_prediction": 6.012995072524063e-06,
"scaled_weighted_latent": 2.642240134318781e-16,
"total": 2.642240134318781e-16
},
"gradients": {
"norms": {
"prediction": 0.00018826978339347988,
"weighted_latent": 1.3986357627615575e-09,
"total": 1.3986357627615575e-09
},
"prediction_weighted_latent_dot": -1.9212446032961655e-13,
"prediction_weighted_latent_cosine": -0.7296211401274257,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.2540962163475342e-07
},
{
"step": 800,
"losses": {
"prediction_mse": 4.151830944465473e-05,
"latent_mse": 1.944334314862503e-14,
"scaled_prediction": 6.012996891513467e-06,
"scaled_weighted_latent": 2.8159324457849516e-16,
"total": 2.8159324457849516e-16
},
"gradients": {
"norms": {
"prediction": 0.00018826867744792253,
"weighted_latent": 1.471699984101349e-09,
"total": 1.471699984101349e-09
},
"prediction_weighted_latent_dot": -2.0089251171872408e-13,
"prediction_weighted_latent_cosine": -0.7250473870008983,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.597900845557888e-07
},
{
"step": 900,
"losses": {
"prediction_mse": 4.1518345824442804e-05,
"latent_mse": 1.809421109309501e-14,
"scaled_prediction": 6.013001893734327e-06,
"scaled_weighted_latent": 2.620541003094698e-16,
"total": 2.620541003094698e-16
},
"gradients": {
"norms": {
"prediction": 0.00018827187886927277,
"weighted_latent": 1.4837001627299173e-09,
"total": 1.4837001627299173e-09
},
"prediction_weighted_latent_dot": -1.7204659463580796e-13,
"prediction_weighted_latent_cosine": -0.6159060638539454,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.0148782431970176e-07
},
{
"step": 1000,
"losses": {
"prediction_mse": 4.151833854848519e-05,
"latent_mse": 2.2932690292641437e-14,
"scaled_prediction": 6.013000984239625e-06,
"scaled_weighted_latent": 3.321286331100056e-16,
"total": 3.321286331100056e-16
},
"gradients": {
"norms": {
"prediction": 0.0001882705109892413,
"weighted_latent": 1.6801082747264218e-09,
"total": 1.6801082747264218e-09
},
"prediction_weighted_latent_dot": -1.919873494123786e-13,
"prediction_weighted_latent_cosine": -0.6069501745436617,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.913382554150303e-07
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "d258f5bc6a790ad9fb7ae1657ed73565f958507c7180d00b83394cb390584ef2"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "fe1854be2dc714421e268893a4a5b66e5f1cdb3ca460feecff3d9c7b0247704a"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "e507f81cee389e9a0b0f12b9f39a92f8e280c0430f9b4f090b0a17f9cd40fed1"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "4b1f5ce0dbff7d1c13ff8f6ad98de19cf7724c14e93c60ff29c4b95c610bb8c8"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "55e0f47977e948cd79a17c6f943e37beaf5ba0919f38111cb4390847eae22c7f"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "b641c160f9124d96c7ab50831dc5f2d0b968c86c0db30c3c314183d3243ffa5b"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "77253c4219569e9dc5916087a4e12e5b8cf9b23b01dead990b0e7352c3d5f7c5"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "d6bef779f38b6a94fc6c878c359f57dfc74089c02e7aea9e6f9c8f5cc69a1469"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "4ea58a82a9c823fad458cdb3c8a1c76714a101dda72e02ee13960eab41b0d6c0"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "7550980e321a2bd7453351e6a8c216e9cd64bc018cee845fd1b672231788be1f"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "11aa8f9755fc7aec1e25e3e7a68f1427745d2c2ddd542cf7a418238615450c19"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "d72fb76a891400e98d3b3de0e8d37bce8f8b9352e6c9f04e76f7a9779a002ce8"
}
],
"final_losses": {
"prediction_mse": 4.151833854848519e-05,
"latent_mse": 2.2932690292641437e-14,
"scaled_prediction": 6.013000984239625e-06,
"scaled_weighted_latent": 3.321286331100056e-16,
"total": 3.321286331100056e-16
},
"training_seconds": 2.756971299997531,
"evaluation_seconds": 0.5017042999970727
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 0,
"arm": "refit",
"status": "passed",
"files": {
"seed-0-refit/result.json": "6f806bfaa84452d1cb3d7fe67b2e6286794bb6f36be206ef15928e348d8f537a",
"seed-0-refit/checkpoint.json": "8adc1ee89716b4e854dfa3bbec62ab6fc88748dff57801c330b170279a9afd29",
"inputs/seed-0-fourier/checkpoint.json": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"inputs/seed-0-fourier/result.json": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"seed-0-refit/diagnostics-arrays.npz": "115f24fd9bb08c96ca7ba5897a19c1dc1d81acc5671784d80d7809436b0bc325"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "a8f9147df2cd38287673388cb25e2506b60a34a7c976861cf3dc668d14779642",
"parent_checkpoint_sha256": "3e6f9e0a7f46ca05b29ef07e2f221649e7707a3d6074c17c857c0a02e8a1f182",
"parent_arrays_sha256": "33df10e27c0564ebac1182007a429ed6e085fc39c3b00c36c058c46eb51d19f9",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 3.8743019104003906e-07,
"max_tolerance_ratio": 0.35905024250805767,
"worst_tolerance_index": [
64,
7
],
"actual_at_worst": 0.0007908046245574951,
"saved_at_worst": 0.0007904171943664551,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.15187023463659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 4.151866596657783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6229368565750986e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444363291142508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 7.444350922014564e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.0001227772822666739
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926477193832397,
"saved_at_worst": 1.2926477193832397,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4581003251396934,
"worst_tolerance_index": [
62,
8
],
"actual_at_worst": -0.0012227296829223633,
"saved_at_worst": -0.0012222155928611755,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 3.652342638815753e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014773458242416
},
"reported": {
"count": 784,
"observable_mse": 3.652340092230588e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014773458242416
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5373180166452665e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 3.181101055815816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004940252751111984
},
"reported": {
"count": 580,
"observable_mse": 3.181101055815816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004940252751111984
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 4.129161607124843e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014772526919842
},
"reported": {
"count": 112,
"observable_mse": 4.129156150156632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005014772526919842
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.4345281951175e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 6.04273664066568e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"reported": {
"count": 92,
"observable_mse": 6.0427275457186624e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005461216438561678
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-11,
"numeric_tolerance_ratio": 9.040318834088458e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.778126847464591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 98,
"observable_mse": 3.7781326682306826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987823776900768
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003417623690158127
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.11751791741699e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0049878195859491825
},
"reported": {
"count": 70,
"observable_mse": 3.1175259209703654e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987821914255619
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.001553465506768622
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.356186400400475e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987818654626608
},
"reported": {
"count": 14,
"observable_mse": 4.356192584964447e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498782517388463
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004349702472956356
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004987813998013735
},
"reported": {
"count": 14,
"observable_mse": 6.503110489575192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00498783215880394
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.816079020500183e-08,
"numeric_tolerance_ratio": 0.012117022670509476
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7222198443487287e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297571886330843
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529756722971797
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0030440218389961885
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.335614383104257e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00529757235199213
},
"reported": {
"count": 70,
"observable_mse": 3.3356191124767065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003652826317988257
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0065819373703562e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297572817653418
},
"reported": {
"count": 14,
"observable_mse": 3.0065733881201595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297566764056683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003957228511153945
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.370883056661114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052975681610405445
},
"reported": {
"count": 14,
"observable_mse": 6.370881601469591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005297568626701832
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003044021561014145
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7371144571807235e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311058834195137
},
"reported": {
"count": 98,
"observable_mse": 3.737111546797678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311060231178999
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0009124017806934365
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.151369674014859e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311061628162861
},
"reported": {
"count": 70,
"observable_mse": 3.151365672238171e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311059299856424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0015206697269865119
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.960085814469494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311049986630678
},
"reported": {
"count": 14,
"observable_mse": 3.960098547395319e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311064422130585
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4435499906539917e-08,
"numeric_tolerance_ratio": 0.009428149153154155
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.442864105338231e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311055574566126
},
"reported": {
"count": 14,
"observable_mse": 6.442856829380617e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005311062093824148
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004257874458583739
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.720241511473432e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"reported": {
"count": 98,
"observable_mse": 3.720237145898864e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681416642266
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.12783413392026e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518278449774
},
"reported": {
"count": 72,
"observable_mse": 3.127835225313902e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005473518744111061
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00030094078470998165
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.02617733925581e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254744619131
},
"reported": {
"count": 14,
"observable_mse": 4.0261500544147566e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005477254278957844
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003008681507163908
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.917762220837176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499614402651787
},
"reported": {
"count": 12,
"observable_mse": 6.917752762092277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005499619524925947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0033047741282601615
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.71916830772534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498461425304413
},
"reported": {
"count": 98,
"observable_mse": 3.719156302395277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498470738530159
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.006009125611987981
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 3.126173032796942e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0054947249591350555
},
"reported": {
"count": 72,
"observable_mse": 3.126166484435089e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00549473287537694
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.916241884231567e-09,
"numeric_tolerance_ratio": 0.005108988937015792
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.025795715278946e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498459096997976
},
"reported": {
"count": 14,
"observable_mse": 4.0257600630866364e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005498471204191446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.210719347000122e-08,
"numeric_tolerance_ratio": 0.007811863060872042
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 6.919406587257981e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055208331905305386
},
"reported": {
"count": 12,
"observable_mse": 6.919389124959707e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005520846229046583
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.30385160446167e-08,
"numeric_tolerance_ratio": 0.008400647652971193
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.75562631234061e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 98,
"observable_mse": 3.755628131330013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 3.164217559969984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217758007347584
},
"reported": {
"count": 70,
"observable_mse": 3.164222289342433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002141989845922839
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0052177575416862965
},
"reported": {
"count": 14,
"observable_mse": 3.9748207200318575e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0024479883953403875
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.49347566650249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217755679041147
},
"reported": {
"count": 14,
"observable_mse": 6.493463297374547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005217761266976595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0036719825930105815
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622227424988523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.184563972055912e-11,
"numeric_tolerance_ratio": 6.156109073935996e-05
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 4.151871689828113e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"reported": {
"count": 84,
"observable_mse": 4.151866596657783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.072111599205138e-05
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.444363291142508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360026644082518e-16
},
"reported": {
"count": 14,
"observable_mse": 7.444350922014564e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360023997104558e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.0001227772822666739
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640167688019574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707476753741503
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005929166943850281
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 2.1136473151273094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005575723946094513
},
"reported": {
"count": 72,
"observable_mse": 2.1136505893082358e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055757262744009495
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0014948300936472665
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.2392847313312814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707480479031801
},
"reported": {
"count": 14,
"observable_mse": 2.239291643491015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005707477685064077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0017787500831550841
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.3784225049894303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441648583859205
},
"reported": {
"count": 12,
"observable_mse": 2.37841213674983e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006441639736294746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.005381193482372771
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2945678234100342,
"saved_at_worst": 1.2945678234100342,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5731212112389529,
"worst_tolerance_index": [
1729,
8
],
"actual_at_worst": -0.0003994107246398926,
"saved_at_worst": -0.0004000067710876465,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 3.751116673811339e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009355312213301659
},
"reported": {
"count": 6272,
"observable_mse": 3.751118129002862e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009355313144624233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00048117153551408443
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 3.3697288017719984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009192988276481628
},
"reported": {
"count": 4092,
"observable_mse": 3.369730256963521e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009192988276481628
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.450304388257766e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 4.0343762520933524e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009355311281979084
},
"reported": {
"count": 896,
"observable_mse": 4.034377707284875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009355312213301659
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00048117155866669024
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 4.7689049097243696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.009854805655777454
},
"reported": {
"count": 1284,
"observable_mse": 4.768906001118012e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.009854805655777454
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0862135916119795e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2790601253509521,
"saved_at_worst": 1.2790601253509521,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.003545761108398e-07,
"max_tolerance_ratio": 0.5058330479705814,
"worst_tolerance_index": [
1528,
4
],
"actual_at_worst": -0.0008991509675979614,
"saved_at_worst": -0.0008997023105621338,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 8.092550706351176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018027817830443382
},
"reported": {
"count": 3136,
"observable_mse": 8.092560165096074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018027817830443382
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.382813912334983e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 7.759645086480305e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017672287300229073
},
"reported": {
"count": 1582,
"observable_mse": 7.759651634842157e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017672287300229073
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.548361852765083e-11,
"numeric_tolerance_ratio": 6.497940101235427e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 8.137684926623479e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018027817830443382
},
"reported": {
"count": 448,
"observable_mse": 8.137695112964138e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018027817830443382
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.00010104116440875068
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 8.550452184863389e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01852450706064701
},
"reported": {
"count": 1106,
"observable_mse": 8.55046309879981e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01852450706064701
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-10,
"numeric_tolerance_ratio": 0.00010821408368344566
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2832953929901123,
"saved_at_worst": 1.2832953929901123,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.491185890233412,
"worst_tolerance_index": [
14,
4
],
"actual_at_worst": 0.002135440707206726,
"saved_at_worst": 0.002134844660758972,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00030053246882744133,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03361130878329277
},
"reported": {
"count": 3136,
"observable_mse": 0.0003005327598657459,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03361130878329277
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002825468462187876
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00030422108829952776,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03238679841160774
},
"reported": {
"count": 1001,
"observable_mse": 0.0003042213211301714,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.032386794686317444
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0008788799261729612
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0002996882249135524,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03361131250858307
},
"reported": {
"count": 448,
"observable_mse": 0.0002996884868480265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03361131250858307
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.6193447411060333e-10,
"numeric_tolerance_ratio": 0.00025431300611185976
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.0002985680184792727,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03431724011898041
},
"reported": {
"count": 1687,
"observable_mse": 0.00029856833862140775,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.03431724011898041
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2014213502407074e-10,
"numeric_tolerance_ratio": 0.00031086081530719425
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2083498239517212,
"saved_at_worst": 1.2083498239517212,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.364418029785156e-07,
"max_tolerance_ratio": 0.40516607583806,
"worst_tolerance_index": [
2102,
7
],
"actual_at_worst": -0.0002973973751068115,
"saved_at_worst": -0.00029781460762023926,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0013533851597458124,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06821805983781815
},
"reported": {
"count": 3136,
"observable_mse": 0.0013533857418224216,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06821805983781815
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.00051268989037383
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0013032231945544481,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06432659178972244
},
"reported": {
"count": 407,
"observable_mse": 0.001303224009461701,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06432659178972244
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.149072527885437e-10,
"numeric_tolerance_ratio": 0.0007209511658854158
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0013566911220550537,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06821805238723755
},
"reported": {
"count": 448,
"observable_mse": 0.0013566919369623065,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06821804493665695
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0009525398650602308
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0013616863870993257,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0688893124461174
},
"reported": {
"count": 2281,
"observable_mse": 0.0013616869691759348,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0688893124461174
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005123153020443514
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128660127520561
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.758500679396093e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7585035897791386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.899485319690801e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010128450579941273
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7913549022050574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.7913512642262504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6242380475881334e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010601093992590904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.960090907639824e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.960088361054659e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.536540241476581e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010598846711218357
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.98074698750861e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.980756446253508e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.42124123167376e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03354682773351669
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00016841452452354133,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001684145099716261,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4310898925389558e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.034877024590969086
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002028434246312827,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00020284306083340198,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-10,
"numeric_tolerance_ratio": 0.00035656520299299306
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.033779799938201904
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018723947869148105,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018723943503573537,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.285336176055767e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020691297948360443
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.837881275918335e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.837849989300594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.00031043304973434846
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 6.998432908560917e-16
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.009684504009783268
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.35564543295186e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.35564543295186e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.622226697392762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.622221240424551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.431860947590584e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0048739295452833176
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.30572659347672e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.3057307771523483e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.1740513894015114e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.1640156774083152e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.1640189515892416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.2671108366151636e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010431777685880661
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.1628161600092426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.162837260286324e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1100277081131935e-10,
"numeric_tolerance_ratio": 0.0002099189932115251
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.72222057194449e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.722221663338132e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0873462981808747e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.857536648865789e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.8575337384827435e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8991992866106846e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7742513086413965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.774241849896498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.423179539860027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.7556277675321326e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.755628131330013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6243670323069163e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.1641156713012606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.1641131247160956e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5360248705162758e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 4.222623465466313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 4.222628194838762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.709486040680649e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 3.188768096151762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 3.1887459044810385e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.219167072325945e-10,
"numeric_tolerance_ratio": 0.0002212113205401971
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1726252927910537e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.172617289237678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.978241468780203e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.0208916945848614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.020887329010293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.343765013792109e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 4.151866596657783e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 6.013048277964117e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 6.013048277964117e-06
},
"gradients": {
"norms": {
"prediction": 0.00018827685562428087,
"weighted_latent": 2.6831793128829187e-18,
"total": 0.00018827685562428087
},
"prediction_weighted_latent_dot": 8.642760077601405e-27,
"prediction_weighted_latent_cosine": 1.710825928725321e-05,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "8adc1ee89716b4e854dfa3bbec62ab6fc88748dff57801c330b170279a9afd29"
}
],
"final_losses": {
"prediction_mse": 4.151866596657783e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 6.013048277964117e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 6.013048277964117e-06
},
"training_seconds": 4.309997893869877e-05,
"evaluation_seconds": 0.6919727000058629
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 1,
"arm": "prediction_latent",
"status": "passed",
"files": {
"seed-1-prediction_latent/result.json": "b9acf2d8ca134acac2d22d8ea5c94f40b34f9c9f8f377fdf91c019b6d88fc5dc",
"seed-1-prediction_latent/checkpoint.json": "8839c5da214cb000de1e7257c4b9ecc65d0f72b253308ad9faea8bf91a8fdc6a",
"inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"seed-1-prediction_latent/diagnostics-arrays.npz": "77a8b37898db44ded5f133b42e8963446d39beabecdba15b4818753f4c494cef"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.6193599700927734e-07,
"max_tolerance_ratio": 0.45109829629777515,
"worst_tolerance_index": [
1,
6
],
"actual_at_worst": -0.00023978948593139648,
"saved_at_worst": -0.00024025142192840576,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676346371648833e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 1.8676328181754798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.815598532142596e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.369931800989434e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 3.369917976669967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.00013777889110753617
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.291385531425476,
"saved_at_worst": 1.291385531425476,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.5049480542195207,
"worst_tolerance_index": [
616,
3
],
"actual_at_worst": 0.0009182542562484741,
"saved_at_worst": 0.0009188055992126465,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 1.4954192010918632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004067175090312958
},
"reported": {
"count": 784,
"observable_mse": 1.4954184734961018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004067175090312958
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.265093259511481e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.2249108294781763e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004047949332743883
},
"reported": {
"count": 580,
"observable_mse": 1.2249111023265868e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004047949332743883
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-12,
"numeric_tolerance_ratio": 2.7251460436744265e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 1.770996823324822e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00406717648729682
},
"reported": {
"count": 112,
"observable_mse": 1.770994822436478e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00406717648729682
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9973510455402024e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 2.8653117624344304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"reported": {
"count": 92,
"observable_mse": 2.8653077606577426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.001776687800884e-11,
"numeric_tolerance_ratio": 3.990343126672343e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.591147156432271e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205137968063354
},
"reported": {
"count": 98,
"observable_mse": 1.591144609847106e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120511934161186
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0013191059629536056
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2873621926701162e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120512865483761
},
"reported": {
"count": 70,
"observable_mse": 1.28735900943866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205123998224735
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00032977647986314834
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.774524753272999e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120515659451485
},
"reported": {
"count": 14,
"observable_mse": 1.7745223885867745e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120510071516037
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003957318410873048
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9266941055539064e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00412051472812891
},
"reported": {
"count": 14,
"observable_mse": 2.9266951969475485e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120514262467623
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003297764363621434
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6111895092763007e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151817861944437
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151880726218224
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004550374085107517
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.384819915983826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815179457888007
},
"reported": {
"count": 70,
"observable_mse": 1.3848208254785277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.0065727623458328764
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4024532902112696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"reported": {
"count": 14,
"observable_mse": 1.402454017807031e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815184347331524
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0030335835414162663
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9517736038542353e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151845801621675
},
"reported": {
"count": 14,
"observable_mse": 2.9517867005779408e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151901680976152
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004044776351031832
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6099873391794972e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 98,
"observable_mse": 1.609983155503869e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410440331324935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0029713781524049092
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2653170415433124e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 70,
"observable_mse": 1.2653120393224526e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104403778910637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0033015311714488147
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.9126227925880812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"reported": {
"count": 14,
"observable_mse": 1.9126269762637094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104407504200935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.007923672718656958
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0307024644571356e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041044000536203384
},
"reported": {
"count": 14,
"observable_mse": 3.0306960979942232e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0026412263323736225
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384033847600222
},
"reported": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384037107229233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022661433552030583
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2924613656650763e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374961368739605
},
"reported": {
"count": 72,
"observable_mse": 1.2924631846544798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374966491013765
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003563329461384987
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7241960449609905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043840352445840836
},
"reported": {
"count": 14,
"observable_mse": 1.724195317365229e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384040366858244
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.00356108160832695
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.179249688400887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0044380780309438705
},
"reported": {
"count": 12,
"observable_mse": 3.179239502060227e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004438071511685848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004515324651932781
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5865916793700308e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355667158961296
},
"reported": {
"count": 98,
"observable_mse": 1.5865951354498975e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355666693300009
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003243745464814044
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2939838597958442e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043465327471494675
},
"reported": {
"count": 72,
"observable_mse": 1.2939861335325986e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004346531815826893
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006491621714371808
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7255477359867655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355668555945158
},
"reported": {
"count": 14,
"observable_mse": 1.725551010167692e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355671815574169
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022706210151850026
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.180122803314589e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004410072695463896
},
"reported": {
"count": 12,
"observable_mse": 3.1801344448467717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00441007548943162
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001938898741991697
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6066007447079755e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 98,
"observable_mse": 1.6065991076175123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838496565819
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0036308001127401722
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2739343219436705e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843153178692
},
"reported": {
"count": 70,
"observable_mse": 1.2739328667521477e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838962227106
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002970654539643297
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.8545022612670437e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107845015823841
},
"reported": {
"count": 14,
"observable_mse": 1.854493712016847e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107835702598095
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00660145605781308
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.022031523869373e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 14,
"observable_mse": 3.0220360713428818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107840359210968
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0023105088575984434
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6136897102114744e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003715944942086935
},
"reported": {
"count": 98,
"observable_mse": 1.6136866179294884e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037159454077482224
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003395036021685274
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.3835786376148462e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037159451749175787
},
"reported": {
"count": 84,
"observable_mse": 1.3835753634339198e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003715945640578866
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003395035964053927
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.994356327690184e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003715943545103073
},
"reported": {
"count": 14,
"observable_mse": 2.9943543268018402e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037159447092562914
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0008487590486448316
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897396527579986e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00388339813798666
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834072183817625
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.006540465866677444
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 6.000454959576018e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964682560414076
},
"reported": {
"count": 72,
"observable_mse": 6.000467237754492e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964771036058664
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006412915697540378
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.797699254297186e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038833962753415108
},
"reported": {
"count": 14,
"observable_mse": 7.79771198722301e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834111765027046
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.010733069131502397
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 1.6882830095710233e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043688067235052586
},
"reported": {
"count": 12,
"observable_mse": 1.6882742784218863e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004368808586150408
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.001296311477784105
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2916944026947021,
"saved_at_worst": 1.2916944026947021,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.487018426439568,
"worst_tolerance_index": [
3093,
3
],
"actual_at_worst": 0.0007083266973495483,
"saved_at_worst": 0.000708848237991333,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 1.5423491277033463e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007828881964087486
},
"reported": {
"count": 6272,
"observable_mse": 1.5423493096022867e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007828881964087486
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-12,
"numeric_tolerance_ratio": 1.8161882069188074e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.2795794646081049e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0077840243466198444
},
"reported": {
"count": 4092,
"observable_mse": 1.2795802831533365e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0077840243466198444
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.185452315956354e-12,
"numeric_tolerance_ratio": 8.174991757688275e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 1.6711635907995515e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007828882895410061
},
"reported": {
"count": 896,
"observable_mse": 1.67116286320379e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007828882895410061
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.263818590310251e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 2.289883923367597e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007970155216753483
},
"reported": {
"count": 1284,
"observable_mse": 2.289882831973955e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.007970155216753483
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.088900188280637e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2809293270111084,
"saved_at_worst": 1.2809293270111084,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5258328609838328,
"worst_tolerance_index": [
1761,
6
],
"actual_at_worst": -0.0004856884479522705,
"saved_at_worst": -0.00048513710498809814,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.089183519477956e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01538029033690691
},
"reported": {
"count": 3136,
"observable_mse": 3.089184610871598e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01538029033690691
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0880325088450617e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 2.921657869592309e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.015239685773849487
},
"reported": {
"count": 1582,
"observable_mse": 2.9216605980764143e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.015239685773849487
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-11,
"numeric_tolerance_ratio": 2.720535623581703e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.089471283601597e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.015380291268229485
},
"reported": {
"count": 448,
"observable_mse": 3.089472374995239e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01538029033690691
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00036694717131001057
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 3.328692037030123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.015579203143715858
},
"reported": {
"count": 1106,
"observable_mse": 3.3286916732322425e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.015579202212393284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0003640936753548345
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2789326906204224,
"saved_at_worst": 1.2789326906204224,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.44440771948250757,
"worst_tolerance_index": [
3009,
12
],
"actual_at_worst": 0.0003948807716369629,
"saved_at_worst": 0.0003944188356399536,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0001303740864386782,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030089888721704483
},
"reported": {
"count": 3136,
"observable_mse": 0.00013037415919825435,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030089886859059334
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.00046461721275974235
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00013071941793896258,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0300093162804842
},
"reported": {
"count": 1001,
"observable_mse": 0.0001307194761466235,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0300093162804842
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.745659136107834e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0001292203669436276,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030089884996414185
},
"reported": {
"count": 448,
"observable_mse": 0.0001292204688070342,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030089884996414185
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.00010056391497475709
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00013047554239165038,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030137594789266586
},
"reported": {
"count": 1687,
"observable_mse": 0.00013047562970314175,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.030137594789266586
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.618696156200086e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.254866123199463,
"saved_at_worst": 1.254866123199463,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.748603820800781e-07,
"max_tolerance_ratio": 0.41287302871708076,
"worst_tolerance_index": [
2452,
3
],
"actual_at_worst": -0.0008269697427749634,
"saved_at_worst": -0.0008274167776107788,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0006314623169600964,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05984501913189888
},
"reported": {
"count": 3136,
"observable_mse": 0.00063146254979074,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05984501540660858
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.000533365233979093
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0006536822183988988,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05909571051597595
},
"reported": {
"count": 407,
"observable_mse": 0.0006536825094372034,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05909571051597595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00027318094406279765
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0006347591406665742,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.059845007956027985
},
"reported": {
"count": 448,
"observable_mse": 0.0006347593734972179,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.059845007956027985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.000218933626494742
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0006268501165322959,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.059977732598781586
},
"reported": {
"count": 2281,
"observable_mse": 0.0006268503493629396,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.059977736324071884
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0005323536447663624
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008089635521173477
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00808099564164877
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.650881858950015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.650880221859552e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6343922773059064e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008295925334095955
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6665429939166643e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6665442672092468e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2711741145488733e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008289745077490807
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5941945093800314e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5941997844493017e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.266673141096683e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.027659518644213676
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.701061935629696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.701080853119493e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8917489796876907e-10,
"numeric_tolerance_ratio": 0.00018735732936814417
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025488151237368584
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.793230906827375e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.793223630869761e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.219692932613361e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025727983564138412
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.805288234725595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.805289689917117e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4424899806819776e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01637626811861992
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.188980554114096e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.188985647284426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0769799138515745e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007163230329751968
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.8452581571182236e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.845252154453192e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.002665031701326e-11,
"numeric_tolerance_ratio": 5.991609002281226e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01430405955761671
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.9898714021546766e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.989876131527126e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.7105778271808575e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0038363952189683914
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.709371286386158e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.709345820534509e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5443691858507307e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897401075053494e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-13,
"numeric_tolerance_ratio": 4.5440247119973885e-07
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008749740198254585
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0526686057564802e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0526651496766135e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.4490002241757354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.611191146366764e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.080316883302202e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5991054169717245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5991059626685455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.448255872186213e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6771049558883533e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.677106411079876e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4527550979481874e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6873334971023723e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6873336790013127e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-12,
"numeric_tolerance_ratio": 1.8159253315753377e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.9213950508856215e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.921396869875025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8155011054046775e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.764368607837241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.7643691535340622e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4473570618693215e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 7.5585703598335385e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 7.558583092759363e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2723308806138392e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.499810635636095e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.4998324635089375e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.181151457204371e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6655649233143777e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6655654690111987e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.447894385869645e-06
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0008061017724685371,
"latent_mse": 0.01724092662334442,
"scaled_prediction": 0.00011674577399389818,
"scaled_weighted_latent": 0.000249696196988225,
"total": 0.00036644196370616555
},
"gradients": {
"norms": {
"prediction": 0.0013187306467443705,
"weighted_latent": 0.0013792344834655523,
"total": 0.0023915665224194527
},
"prediction_weighted_latent_dot": 1.0391254363639746e-06,
"prediction_weighted_latent_cosine": 0.571312557453286,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0008706919034011662,
"latent_mse": 0.01624198444187641,
"scaled_prediction": 0.0001261002034880221,
"scaled_weighted_latent": 0.00023522874107584357,
"total": 0.00036132894456386566
},
"gradients": {
"norms": {
"prediction": 0.0017319160979241133,
"weighted_latent": 0.0013384808553382754,
"total": 0.0026371318381279707
},
"prediction_weighted_latent_dot": 1.0817008160302066e-06,
"prediction_weighted_latent_cosine": 0.46662515227534174,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.029491163790225983
},
{
"step": 100,
"losses": {
"prediction_mse": 1.2324222552706487e-05,
"latent_mse": 7.64099313528277e-05,
"scaled_prediction": 1.7848874449555296e-06,
"scaled_weighted_latent": 1.106626655200671e-06,
"total": 2.8915142138430383e-06
},
"gradients": {
"norms": {
"prediction": 5.4994736274238676e-05,
"weighted_latent": 9.689817670732737e-05,
"total": 0.00012219359632581472
},
"prediction_weighted_latent_dot": 1.2588040609884388e-09,
"prediction_weighted_latent_cosine": 0.2362225788337237,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0007766924099996686
},
{
"step": 200,
"losses": {
"prediction_mse": 1.3779517757939175e-05,
"latent_mse": 1.353805600956548e-05,
"scaled_prediction": 1.9956544292654144e-06,
"scaled_weighted_latent": 1.9606839885000227e-07,
"total": 2.1917228423262713e-06
},
"gradients": {
"norms": {
"prediction": 3.913431282853708e-05,
"weighted_latent": 3.868190469802357e-05,
"total": 5.253093036117207e-07
},
"prediction_weighted_latent_dot": -1.5137540110288228e-09,
"prediction_weighted_latent_cosine": -0.9999763849457488,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 6.9785883169970475e-06
},
{
"step": 300,
"losses": {
"prediction_mse": 1.3765730727754999e-05,
"latent_mse": 1.3676089110958856e-05,
"scaled_prediction": 1.993657633647672e-06,
"scaled_weighted_latent": 1.980674966262086e-07,
"total": 2.1917251160630258e-06
},
"gradients": {
"norms": {
"prediction": 3.886024933308363e-05,
"weighted_latent": 3.890350490109995e-05,
"total": 1.0165747426071903e-06
},
"prediction_weighted_latent_dot": -1.5112843199105441e-09,
"prediction_weighted_latent_cosine": -0.9996589624877547,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.979640991427004e-05
},
{
"step": 400,
"losses": {
"prediction_mse": 1.3768789358437061e-05,
"latent_mse": 1.3674572983291e-05,
"scaled_prediction": 1.9941005575674353e-06,
"scaled_weighted_latent": 1.980455408556736e-07,
"total": 2.192145984736271e-06
},
"gradients": {
"norms": {
"prediction": 3.891400410793722e-05,
"weighted_latent": 3.8907051930436864e-05,
"total": 5.7175780057150405e-06
},
"prediction_weighted_latent_dot": -1.4976839768365835e-09,
"prediction_weighted_latent_cosine": -0.9892041698781848,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0003822461876552552
},
{
"step": 500,
"losses": {
"prediction_mse": 1.3779789696855005e-05,
"latent_mse": 1.3661119737662375e-05,
"scaled_prediction": 1.995693764911266e-06,
"scaled_weighted_latent": 1.978507100375282e-07,
"total": 2.1935445602139225e-06
},
"gradients": {
"norms": {
"prediction": 4.00488315790426e-05,
"weighted_latent": 3.890451989718713e-05,
"total": 1.3405261597654317e-05
},
"prediction_weighted_latent_dot": -1.4688850136224119e-09,
"prediction_weighted_latent_cosine": -0.9427529272839232,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005967266042716801
},
{
"step": 600,
"losses": {
"prediction_mse": 1.376701857225271e-05,
"latent_mse": 1.3671409760718234e-05,
"scaled_prediction": 1.9938440800615354e-06,
"scaled_weighted_latent": 1.979997250600718e-07,
"total": 2.191843805121607e-06
},
"gradients": {
"norms": {
"prediction": 3.883298268192448e-05,
"weighted_latent": 3.890189327648841e-05,
"total": 3.6848484796792036e-06
},
"prediction_weighted_latent_dot": -1.5038899014996332e-09,
"prediction_weighted_latent_cosine": -0.99550754500707,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00015977371367625892
},
{
"step": 700,
"losses": {
"prediction_mse": 1.3792460777040105e-05,
"latent_mse": 1.370991321891779e-05,
"scaled_prediction": 1.9975288978457684e-06,
"scaled_weighted_latent": 1.9855737320995104e-07,
"total": 2.1960863705317024e-06
},
"gradients": {
"norms": {
"prediction": 4.25968537456356e-05,
"weighted_latent": 3.8942300307098776e-05,
"total": 1.801659345801454e-05
},
"prediction_weighted_latent_dot": -1.5031985656221991e-09,
"prediction_weighted_latent_cosine": -0.9061857496691742,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0010927342809736729
},
{
"step": 800,
"losses": {
"prediction_mse": 1.3770555597147904e-05,
"latent_mse": 1.3717112778977025e-05,
"scaled_prediction": 1.994356352952309e-06,
"scaled_weighted_latent": 1.9866163825099648e-07,
"total": 2.193017962781596e-06
},
"gradients": {
"norms": {
"prediction": 4.083912426722236e-05,
"weighted_latent": 3.893457324011251e-05,
"total": 1.1470227946119849e-05
},
"prediction_weighted_latent_dot": -1.5260845920295196e-09,
"prediction_weighted_latent_cosine": -0.9597691098227897,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00030544379842467606
},
{
"step": 900,
"losses": {
"prediction_mse": 1.3786719136987813e-05,
"latent_mse": 1.3740334907197393e-05,
"scaled_prediction": 1.9966973923146725e-06,
"scaled_weighted_latent": 1.9899795233868645e-07,
"total": 2.19569528780994e-06
},
"gradients": {
"norms": {
"prediction": 4.388808520161547e-05,
"weighted_latent": 3.902985918102786e-05,
"total": 2.101091740769334e-05
},
"prediction_weighted_latent_dot": -1.5040174661251626e-09,
"prediction_weighted_latent_cosine": -0.8780298122504341,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0008061519474722445
},
{
"step": 1000,
"losses": {
"prediction_mse": 1.3835753634339198e-05,
"latent_mse": 1.3808251424052287e-05,
"scaled_prediction": 2.003798954319791e-06,
"scaled_weighted_latent": 1.9998157085865387e-07,
"total": 2.203780468335026e-06
},
"gradients": {
"norms": {
"prediction": 5.0987840950256214e-05,
"weighted_latent": 3.904912227881141e-05,
"total": 3.4724351280601695e-05
},
"prediction_weighted_latent_dot": -1.4594064845496746e-09,
"prediction_weighted_latent_cosine": -0.732990545077887,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.002056598663330078
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "89d8bc5ffec17a479dca72c2881e928d8e51bc09264d06412d3f75d556c67e74"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "7f42839fc28475d9ffa07003e56fad1804e51cf3b4be6b6e04200f48c186456e"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "ea8c90d58857fa07b3884aacfe5d8ec9dddd3d6b71ed9c956285ef82f083164f"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "b0f9b3b30a10095087dddac9300d70691ce6aa8ffc51f561d2970b3fed180937"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "1bba1bfc2961bd7f9a18824f1962222da0ae93b371f14697851cdab7a9cc63a2"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "d1deba08d32aa677cc5bfd6ac9abb98ce005268fa5b44f7684a14a5573620f15"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "4f4cd928216f757574214682a35b84bbe4cd4905ba3d4684c7c5d8209a3bf4fa"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "b0f88e8adc771a2547231245718b6a87d5af804ed5c3063e4f244ed935960ea7"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "aacf0c12fcddd8301ba9ef60f73597804bb76fef9ac6da035265d79c289c1319"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "bde9444c65c61fd4f7ce4fb4724126b712c0b6c56aa754a7e9cdb1c944d890cc"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "7272aa472e41b04e8c81c1a1279276555ef4b7cf9d430f8dbddb0d620e453738"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "8839c5da214cb000de1e7257c4b9ecc65d0f72b253308ad9faea8bf91a8fdc6a"
}
],
"final_losses": {
"prediction_mse": 1.3835753634339198e-05,
"latent_mse": 1.3808251424052287e-05,
"scaled_prediction": 2.003798954319791e-06,
"scaled_weighted_latent": 1.9998157085865387e-07,
"total": 2.203780468335026e-06
},
"training_seconds": 3.2491507999948226,
"evaluation_seconds": 0.5362058999598958
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 1,
"arm": "latent_only",
"status": "passed",
"files": {
"seed-1-latent_only/result.json": "9c59b3e4a5eb6a983a8866e9fbb5b08bfb3fe78276741c06bff9b2aa83c9ecc6",
"seed-1-latent_only/checkpoint.json": "b23dac4e55edfd0ed3c5276331b5fd666cf6cb758202e1d9e5efe7d2d1aea453",
"inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"seed-1-latent_only/diagnostics-arrays.npz": "846787664b0b27fac1ba45d574fb7d6cb7e9292bce0062d62f660bd361f1039f"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.6193599700927734e-07,
"max_tolerance_ratio": 0.45109829629777515,
"worst_tolerance_index": [
1,
6
],
"actual_at_worst": -0.00023978948593139648,
"saved_at_worst": -0.00024025142192840576,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676346371648833e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 1.8676328181754798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.815598532142596e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.369931800989434e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 3.369917976669967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.00013777889110753617
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.291385531425476,
"saved_at_worst": 1.291385531425476,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.42046233387194104,
"worst_tolerance_index": [
87,
6
],
"actual_at_worst": 0.0009859204292297363,
"saved_at_worst": 0.0009863823652267456,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 1.553986658109352e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003849141066893935
},
"reported": {
"count": 784,
"observable_mse": 1.5539857486146502e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003849140601232648
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00033623839970712183
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.2950129530509003e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037928984966129065
},
"reported": {
"count": 580,
"observable_mse": 1.29501286210143e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00379289872944355
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001688047220682108
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 1.817941119952593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038491422310471535
},
"reported": {
"count": 112,
"observable_mse": 1.8179387552663684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038491422310471535
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3603951707509615e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 2.8653117624344304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"reported": {
"count": 92,
"observable_mse": 2.8653077606577426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.001776687800884e-11,
"numeric_tolerance_ratio": 3.990343126672343e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.591147156432271e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205137968063354
},
"reported": {
"count": 98,
"observable_mse": 1.591144609847106e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120511934161186
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0013191059629536056
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2873621926701162e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120512865483761
},
"reported": {
"count": 70,
"observable_mse": 1.28735900943866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205123998224735
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00032977647986314834
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.774524753272999e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120515659451485
},
"reported": {
"count": 14,
"observable_mse": 1.7745223885867745e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120510071516037
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003957318410873048
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9266941055539064e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00412051472812891
},
"reported": {
"count": 14,
"observable_mse": 2.9266951969475485e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120514262467623
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003297764363621434
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6111895092763007e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151817861944437
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151880726218224
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004550374085107517
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.384819915983826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815179457888007
},
"reported": {
"count": 70,
"observable_mse": 1.3848208254785277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.0065727623458328764
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4024532902112696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"reported": {
"count": 14,
"observable_mse": 1.402454017807031e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815184347331524
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0030335835414162663
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9517736038542353e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151845801621675
},
"reported": {
"count": 14,
"observable_mse": 2.9517867005779408e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151901680976152
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004044776351031832
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6099873391794972e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 98,
"observable_mse": 1.609983155503869e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410440331324935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0029713781524049092
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2653170415433124e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 70,
"observable_mse": 1.2653120393224526e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104403778910637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0033015311714488147
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.9126227925880812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"reported": {
"count": 14,
"observable_mse": 1.9126269762637094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104407504200935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.007923672718656958
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0307024644571356e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041044000536203384
},
"reported": {
"count": 14,
"observable_mse": 3.0306960979942232e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0026412263323736225
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384033847600222
},
"reported": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384037107229233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022661433552030583
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2924613656650763e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374961368739605
},
"reported": {
"count": 72,
"observable_mse": 1.2924631846544798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374966491013765
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003563329461384987
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7241960449609905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043840352445840836
},
"reported": {
"count": 14,
"observable_mse": 1.724195317365229e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384040366858244
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.00356108160832695
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.179249688400887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0044380780309438705
},
"reported": {
"count": 12,
"observable_mse": 3.179239502060227e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004438071511685848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004515324651932781
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5865916793700308e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355667158961296
},
"reported": {
"count": 98,
"observable_mse": 1.5865951354498975e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355666693300009
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003243745464814044
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2939838597958442e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043465327471494675
},
"reported": {
"count": 72,
"observable_mse": 1.2939861335325986e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004346531815826893
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006491621714371808
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7255477359867655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355668555945158
},
"reported": {
"count": 14,
"observable_mse": 1.725551010167692e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355671815574169
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022706210151850026
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.180122803314589e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004410072695463896
},
"reported": {
"count": 12,
"observable_mse": 3.1801344448467717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00441007548943162
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001938898741991697
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6066007447079755e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 98,
"observable_mse": 1.6065991076175123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838496565819
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0036308001127401722
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2739343219436705e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843153178692
},
"reported": {
"count": 70,
"observable_mse": 1.2739328667521477e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838962227106
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002970654539643297
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.8545022612670437e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107845015823841
},
"reported": {
"count": 14,
"observable_mse": 1.854493712016847e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107835702598095
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00660145605781308
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.022031523869373e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 14,
"observable_mse": 3.0220360713428818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107840359210968
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0023105088575984434
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.082230093947146e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.6341662956165237e-07
},
"reported": {
"count": 98,
"observable_mse": 2.0822268197662197e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.6449087070213864e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.074241140486265e-09,
"numeric_tolerance_ratio": 0.0010742234704908661
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676168110687286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.630751853554102e-07
},
"reported": {
"count": 84,
"observable_mse": 1.867613900685683e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.6483070908179798e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7555237263877643e-09,
"numeric_tolerance_ratio": 0.0017554947904426545
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.369911064510234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.6545044445592794e-07
},
"reported": {
"count": 14,
"observable_mse": 3.3699041523505e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.6243686218331277e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.0135822726151673e-09,
"numeric_tolerance_ratio": 0.0030135333217254806
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897396527579986e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00388339813798666
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834072183817625
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.006540465866677444
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 6.000454959576018e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964682560414076
},
"reported": {
"count": 72,
"observable_mse": 6.000467237754492e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964771036058664
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006412915697540378
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.797699254297186e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038833962753415108
},
"reported": {
"count": 14,
"observable_mse": 7.79771198722301e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834111765027046
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.010733069131502397
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 1.6882830095710233e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043688067235052586
},
"reported": {
"count": 12,
"observable_mse": 1.6882742784218863e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004368808586150408
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.001296311477784105
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2916944026947021,
"saved_at_worst": 1.2916944026947021,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.487018426439568,
"worst_tolerance_index": [
3093,
3
],
"actual_at_worst": 0.0007083266973495483,
"saved_at_worst": 0.000708848237991333,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 1.5348035958595574e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00719467643648386
},
"reported": {
"count": 6272,
"observable_mse": 1.534803413960617e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00719467643648386
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-12,
"numeric_tolerance_ratio": 1.8162018906835934e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.2756069736497011e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0070650274865329266
},
"reported": {
"count": 4092,
"observable_mse": 1.275607428397052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0070650274865329266
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-12,
"numeric_tolerance_ratio": 4.541680107981497e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 1.6645597497699782e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007194678299129009
},
"reported": {
"count": 896,
"observable_mse": 1.664558658376336e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007194678299129009
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0895799723505444e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 2.2702946807839908e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007593102287501097
},
"reported": {
"count": 1284,
"observable_mse": 2.2702939531882294e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.007593102287501097
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2594764686533295e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2837709188461304,
"saved_at_worst": 1.2837709188461304,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5258328609838328,
"worst_tolerance_index": [
1761,
6
],
"actual_at_worst": -0.0004856884479522705,
"saved_at_worst": -0.00048513710498809814,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 2.7006264645024203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013975739479064941
},
"reported": {
"count": 3136,
"observable_mse": 2.700626828300301e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013975738547742367
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00038844374815022117
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 2.4990224119392224e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013708249665796757
},
"reported": {
"count": 1582,
"observable_mse": 2.4990240490296856e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013708249665796757
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6330095330957698e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 2.7068224881077185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013975740410387516
},
"reported": {
"count": 448,
"observable_mse": 2.7068224881077185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013975740410387516
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 2.986486470035743e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.014349685981869698
},
"reported": {
"count": 1106,
"observable_mse": 2.9864857424399815e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.014349685050547123
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.000382478283674783
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2804754972457886,
"saved_at_worst": 1.2804754972457886,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4426357756470472,
"worst_tolerance_index": [
1822,
8
],
"actual_at_worst": 9.983032941818237e-05,
"saved_at_worst": 9.938329458236694e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 9.646726539358497e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026145141571760178
},
"reported": {
"count": 3136,
"observable_mse": 9.646730177337304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026145141571760178
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.603219520606705e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 9.084730118047446e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02528628334403038
},
"reported": {
"count": 1001,
"observable_mse": 9.084734483622015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02528628334403038
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.326271540262718e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 9.527995280222967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026145141571760178
},
"reported": {
"count": 448,
"observable_mse": 9.527997463010252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026145141571760178
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.162185981706769e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00010011722042690963,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026641670614480972
},
"reported": {
"count": 1687,
"observable_mse": 0.00010011727863457054,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026641670614480972
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.7630678246279204e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2577427625656128,
"saved_at_worst": 1.2577427625656128,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.556510925292969e-07,
"max_tolerance_ratio": 0.43638591490078654,
"worst_tolerance_index": [
2430,
6
],
"actual_at_worst": -0.0014396607875823975,
"saved_at_worst": -0.0014391615986824036,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00048254439025186,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05308973416686058
},
"reported": {
"count": 3136,
"observable_mse": 0.0004825446230825037,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05308973416686058
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00022211271406484442
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0004555753548629582,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.050318289548158646
},
"reported": {
"count": 407,
"observable_mse": 0.0004555756750050932,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.050318289548158646
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2014213502407074e-10,
"numeric_tolerance_ratio": 0.00030619273866421015
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00048519056872464716,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05308973789215088
},
"reported": {
"count": 448,
"observable_mse": 0.00048519091797061265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05308973789215088
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.0003330849845397
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.00048683679779060185,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05356917530298233
},
"reported": {
"count": 2281,
"observable_mse": 0.00048683705972507596,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05356917157769203
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0005860215267881844
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008089635521173477
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00808099564164877
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.650881858950015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.650880221859552e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6343922773059064e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008295925334095955
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6665429939166643e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6665442672092468e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2711741145488733e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008289745077490807
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5941945093800314e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5941997844493017e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.266673141096683e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.027659518644213676
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.701061935629696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.701080853119493e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8917489796876907e-10,
"numeric_tolerance_ratio": 0.00018735732936814417
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025488151237368584
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.793230906827375e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.793223630869761e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.219692932613361e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025727983564138412
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.805288234725595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.805289689917117e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4424899806819776e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01637626811861992
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.188980554114096e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.188985647284426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0769799138515745e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 2.802309211347165e-07
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822144506382756e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.082210994558409e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.448898532294068e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007543846964836121
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.959134533535689e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.959132350748405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1785192766633875e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0038363952189683914
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.709371286386158e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.709345820534509e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5443691858507307e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897401075053494e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-13,
"numeric_tolerance_ratio": 4.5440247119973885e-07
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008749740198254585
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0526686057564802e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0526651496766135e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.4490002241757354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.611191146366764e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.080316883302202e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5991054169717245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5991059626685455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.448255872186213e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6771049558883533e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.677106411079876e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4527550979481874e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.606598380021751e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6065961972344667e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1792860515718863e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.812363916542381e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.8123662812286057e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3604083001964056e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.764368607837241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.7643691535340622e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4473570618693215e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 7.5585703598335385e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 7.558583092759363e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2723308806138392e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.499810635636095e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.4998324635089375e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.181151457204371e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6655649233143777e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6655654690111987e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.447894385869645e-06
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0008061017724685371,
"latent_mse": 0.01724092662334442,
"scaled_prediction": 0.00011674577399389818,
"scaled_weighted_latent": 0.000249696196988225,
"total": 0.000249696196988225
},
"gradients": {
"norms": {
"prediction": 0.0013187306467443705,
"weighted_latent": 0.0013792344834655523,
"total": 0.0013792344834655523
},
"prediction_weighted_latent_dot": 1.0391254363639746e-06,
"prediction_weighted_latent_cosine": 0.571312557453286,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0008845120901241899,
"latent_mse": 0.016194390133023262,
"scaled_prediction": 0.0001281017466681078,
"scaled_weighted_latent": 0.0002345394459553063,
"total": 0.0002345394459553063
},
"gradients": {
"norms": {
"prediction": 0.0017962348647415638,
"weighted_latent": 0.0013357902644202113,
"total": 0.0013357902644202113
},
"prediction_weighted_latent_dot": 1.1284744232398225e-06,
"prediction_weighted_latent_cosine": 0.4703166184558535,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.03137466311454773
},
{
"step": 100,
"losses": {
"prediction_mse": 2.6100775357917883e-05,
"latent_mse": 4.739292853628285e-05,
"scaled_prediction": 3.7801123653480317e-06,
"scaled_weighted_latent": 6.863803605483554e-07,
"total": 6.863803605483554e-07
},
"gradients": {
"norms": {
"prediction": 0.0003757460799533874,
"weighted_latent": 8.124136365950108e-05,
"total": 8.124136365950108e-05
},
"prediction_weighted_latent_dot": 1.79601880034852e-08,
"prediction_weighted_latent_cosine": 0.5883546842546467,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0007346367347054183
},
{
"step": 200,
"losses": {
"prediction_mse": 1.8727765564108267e-05,
"latent_mse": 8.353190361631846e-10,
"scaled_prediction": 2.7122971459903056e-06,
"scaled_weighted_latent": 1.2097723864346044e-11,
"total": 1.2097723864346044e-11
},
"gradients": {
"norms": {
"prediction": 0.0001373724517179653,
"weighted_latent": 3.3851549119390256e-07,
"total": 3.3851549119390256e-07
},
"prediction_weighted_latent_dot": 3.571216586339965e-11,
"prediction_weighted_latent_cosine": 0.7679589267440738,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.8460998489899794e-06
},
{
"step": 300,
"losses": {
"prediction_mse": 1.867616083472967e-05,
"latent_mse": 1.5077970813635704e-14,
"scaled_prediction": 2.7048233732784865e-06,
"scaled_weighted_latent": 2.1837061275120558e-16,
"total": 2.1837061275120558e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575522461906075,
"weighted_latent": 1.2317837860820191e-09,
"total": 1.2317837860820191e-09
},
"prediction_weighted_latent_dot": -1.1001419773001148e-13,
"prediction_weighted_latent_cosine": -0.6578966881980206,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 8.404920492921519e-08
},
{
"step": 400,
"losses": {
"prediction_mse": 1.8676202671485953e-05,
"latent_mse": 1.4480321306712014e-14,
"scaled_prediction": 2.7048295123677235e-06,
"scaled_weighted_latent": 2.097150080563405e-16,
"total": 2.097150080563405e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575679622590542,
"weighted_latent": 1.1411173117537032e-09,
"total": 1.1411173117537032e-09
},
"prediction_weighted_latent_dot": -7.508601487966893e-14,
"prediction_weighted_latent_cosine": -0.4846934833921122,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.6925260126754438e-07
},
{
"step": 500,
"losses": {
"prediction_mse": 1.867614992079325e-05,
"latent_mse": 1.6872303386242585e-14,
"scaled_prediction": 2.7048217816627584e-06,
"scaled_weighted_latent": 2.443574909287389e-16,
"total": 2.443574909287389e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575528282672167,
"weighted_latent": 1.2982581676368454e-09,
"total": 1.2982581676368454e-09
},
"prediction_weighted_latent_dot": -1.0371863551393926e-13,
"prediction_weighted_latent_cosine": -0.5884898711148955,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.282730946490119e-07
},
{
"step": 600,
"losses": {
"prediction_mse": 1.8676179024623707e-05,
"latent_mse": 1.7656993014513074e-14,
"scaled_prediction": 2.7048258743889164e-06,
"scaled_weighted_latent": 2.5572197316337225e-16,
"total": 2.5572197316337225e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575544289778918,
"weighted_latent": 1.320952680572418e-09,
"total": 1.320952680572418e-09
},
"prediction_weighted_latent_dot": -1.0743573457087929e-13,
"prediction_weighted_latent_cosine": -0.5991067960757013,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.4546375243517105e-07
},
{
"step": 700,
"losses": {
"prediction_mse": 1.8676191757549532e-05,
"latent_mse": 1.64592867330296e-14,
"scaled_prediction": 2.7048279207519954e-06,
"scaled_weighted_latent": 2.3837587660412705e-16,
"total": 2.3837587660412705e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575569028034806,
"weighted_latent": 1.238080749033088e-09,
"total": 1.238080749033088e-09
},
"prediction_weighted_latent_dot": -8.547593785112467e-14,
"prediction_weighted_latent_cosine": -0.508553750463921,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.2718136822040833e-07
},
{
"step": 800,
"losses": {
"prediction_mse": 1.867614992079325e-05,
"latent_mse": 1.865822830981502e-14,
"scaled_prediction": 2.7048217816627584e-06,
"scaled_weighted_latent": 2.7022262135499423e-16,
"total": 2.7022262135499423e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575420598499477,
"weighted_latent": 1.422379436455401e-09,
"total": 1.422379436455401e-09
},
"prediction_weighted_latent_dot": -1.1380973203786726e-13,
"prediction_weighted_latent_cosine": -0.5894006926123972,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.1941859763028333e-07
},
{
"step": 900,
"losses": {
"prediction_mse": 1.8676177205634303e-05,
"latent_mse": 1.746145207050763e-14,
"scaled_prediction": 2.704825647015241e-06,
"scaled_weighted_latent": 2.528899979135663e-16,
"total": 2.528899979135663e-16
},
"gradients": {
"norms": {
"prediction": 0.0001357544242637232,
"weighted_latent": 1.3941990895105505e-09,
"total": 1.3941990895105505e-09
},
"prediction_weighted_latent_dot": -1.1605610407156713e-13,
"prediction_weighted_latent_cosine": -0.6131817216372834,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.4708339196877205e-07
},
{
"step": 1000,
"losses": {
"prediction_mse": 1.867613900685683e-05,
"latent_mse": 2.7169164541849394e-14,
"scaled_prediction": 2.7048201900470303e-06,
"scaled_weighted_latent": 3.934844704959957e-16,
"total": 3.934844704959957e-16
},
"gradients": {
"norms": {
"prediction": 0.00013575352204497904,
"weighted_latent": 1.9588655142399602e-09,
"total": 1.9588655142399602e-09
},
"prediction_weighted_latent_dot": -1.1047034191276686e-13,
"prediction_weighted_latent_cosine": -0.41542245860004423,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.4463152448770416e-07
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "0019be82f6c82ad2528fa20d684da6e7c3a34adf8757608d6bac0ed0628f84aa"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "cba955951f5dcde7b0f5c797baacd4d3fb058da239bf7ca514b6afa2c38e3abc"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "6657da4d89be5f5567c70b162592681239a2511b88a0fcc29da8dde9ad7e0132"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "6f03faf58ef0cf71999afed7d14cedffe5f59797c8dcd912b132c44c882191f9"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "f97a1925abbdaf55021492c08c5ef775fffa51dbff05d06b6e75ee134d0145a9"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "df0d72df6257828cc0819af9bacd2c9df95eaaa6a74baccdba5db2e53dc9b5a8"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "08780c1646a18a11d27d727fdbb2f939f9d192ce79c588f5489fe9af0181c688"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "9cbcc1a4b0d82c2f61d1830a66a11492fd695daa5af016f09de1fd7cae98be55"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "0c77d8e63f912c8fb6a623325cdc4fedb286245b917124d90252cee318b02d5e"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "550d7d9030a3f2b73c7b47825309a1ad1c78e3a75816c83d299127f4d7a2bb57"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "b2e4941568ee67bde30dfa33cba010547a395da32e84b7b8de2726ade7ce4a05"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "b23dac4e55edfd0ed3c5276331b5fd666cf6cb758202e1d9e5efe7d2d1aea453"
}
],
"final_losses": {
"prediction_mse": 1.867613900685683e-05,
"latent_mse": 2.7169164541849394e-14,
"scaled_prediction": 2.7048201900470303e-06,
"scaled_weighted_latent": 3.934844704959957e-16,
"total": 3.934844704959957e-16
},
"training_seconds": 2.7909303000196815,
"evaluation_seconds": 0.519522299990058
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 1,
"arm": "refit",
"status": "passed",
"files": {
"seed-1-refit/result.json": "29f717c7e993f537bed0c71ed1e26fb24ecdb13f87d0927b63e256d4ad9006e2",
"seed-1-refit/checkpoint.json": "f27c7aa4ae44e1bd2421dfb90baa66d8366a995f651d65613378922892ca067a",
"inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"seed-1-refit/diagnostics-arrays.npz": "27aa56e1165bd249fe85280a3393502cf51ea898344f423e11966d15c5a0481b"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.6193599700927734e-07,
"max_tolerance_ratio": 0.45109829629777515,
"worst_tolerance_index": [
1,
6
],
"actual_at_worst": -0.00023978948593139648,
"saved_at_worst": -0.00024025142192840576,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676346371648833e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 1.8676328181754798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.815598532142596e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.369931800989434e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 3.369917976669967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.00013777889110753617
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.291385531425476,
"saved_at_worst": 1.291385531425476,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.45109829629777515,
"worst_tolerance_index": [
588,
6
],
"actual_at_worst": -0.00023978948593139648,
"saved_at_worst": -0.00024025142192840576,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 1.5539890227955766e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003849141066893935
},
"reported": {
"count": 784,
"observable_mse": 1.5539879314019345e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003849140601232648
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00033623839970712183
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.2950155905855354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037928984966129065
},
"reported": {
"count": 580,
"observable_mse": 1.2950153177371249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00379289872944355
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001688047220682108
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 1.8179434846388176e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038491422310471535
},
"reported": {
"count": 112,
"observable_mse": 1.817940028558951e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038491422310471535
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.4498083220975305e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 2.8653117624344304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"reported": {
"count": 92,
"observable_mse": 2.8653077606577426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.001776687800884e-11,
"numeric_tolerance_ratio": 3.990343126672343e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.591147156432271e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205137968063354
},
"reported": {
"count": 98,
"observable_mse": 1.591144609847106e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120511934161186
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0013191059629536056
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2873621926701162e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120512865483761
},
"reported": {
"count": 70,
"observable_mse": 1.28735900943866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205123998224735
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00032977647986314834
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.774524753272999e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120515659451485
},
"reported": {
"count": 14,
"observable_mse": 1.7745223885867745e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120510071516037
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003957318410873048
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9266941055539064e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00412051472812891
},
"reported": {
"count": 14,
"observable_mse": 2.9266951969475485e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120514262467623
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003297764363621434
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6111895092763007e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151817861944437
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151880726218224
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004550374085107517
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.384819915983826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815179457888007
},
"reported": {
"count": 70,
"observable_mse": 1.3848208254785277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.0065727623458328764
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4024532902112696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"reported": {
"count": 14,
"observable_mse": 1.402454017807031e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815184347331524
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0030335835414162663
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9517736038542353e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151845801621675
},
"reported": {
"count": 14,
"observable_mse": 2.9517867005779408e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151901680976152
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004044776351031832
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6099873391794972e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 98,
"observable_mse": 1.609983155503869e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410440331324935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0029713781524049092
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2653170415433124e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 70,
"observable_mse": 1.2653120393224526e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104403778910637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0033015311714488147
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.9126227925880812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"reported": {
"count": 14,
"observable_mse": 1.9126269762637094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104407504200935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.007923672718656958
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0307024644571356e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041044000536203384
},
"reported": {
"count": 14,
"observable_mse": 3.0306960979942232e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0026412263323736225
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384033847600222
},
"reported": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384037107229233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022661433552030583
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2924613656650763e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374961368739605
},
"reported": {
"count": 72,
"observable_mse": 1.2924631846544798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374966491013765
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003563329461384987
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7241960449609905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043840352445840836
},
"reported": {
"count": 14,
"observable_mse": 1.724195317365229e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384040366858244
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.00356108160832695
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.179249688400887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0044380780309438705
},
"reported": {
"count": 12,
"observable_mse": 3.179239502060227e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004438071511685848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004515324651932781
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5865916793700308e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355667158961296
},
"reported": {
"count": 98,
"observable_mse": 1.5865951354498975e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355666693300009
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003243745464814044
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2939838597958442e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043465327471494675
},
"reported": {
"count": 72,
"observable_mse": 1.2939861335325986e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004346531815826893
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006491621714371808
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7255477359867655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355668555945158
},
"reported": {
"count": 14,
"observable_mse": 1.725551010167692e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355671815574169
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022706210151850026
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.180122803314589e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004410072695463896
},
"reported": {
"count": 12,
"observable_mse": 3.1801344448467717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00441007548943162
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001938898741991697
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6066007447079755e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 98,
"observable_mse": 1.6065991076175123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838496565819
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0036308001127401722
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2739343219436705e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843153178692
},
"reported": {
"count": 70,
"observable_mse": 1.2739328667521477e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838962227106
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002970654539643297
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.8545022612670437e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107845015823841
},
"reported": {
"count": 14,
"observable_mse": 1.854493712016847e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107835702598095
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00660145605781308
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.022031523869373e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 14,
"observable_mse": 3.0220360713428818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107840359210968
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0023105088575984434
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822486476390623e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.8198777474462986e-11,
"numeric_tolerance_ratio": 3.811940354359698e-05
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676348190638237e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"reported": {
"count": 84,
"observable_mse": 1.8676328181754798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9971583853568556e-05
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.3699314371915534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360026644082518e-16
},
"reported": {
"count": 14,
"observable_mse": 3.369917976669967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360023997104558e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3460521586239338e-10,
"numeric_tolerance_ratio": 0.0001341531308152326
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897396527579986e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00388339813798666
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834072183817625
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.006540465866677444
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 6.000454959576018e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964682560414076
},
"reported": {
"count": 72,
"observable_mse": 6.000467237754492e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964771036058664
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006412915697540378
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.797699254297186e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038833962753415108
},
"reported": {
"count": 14,
"observable_mse": 7.79771198722301e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834111765027046
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.010733069131502397
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 1.6882830095710233e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043688067235052586
},
"reported": {
"count": 12,
"observable_mse": 1.6882742784218863e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004368808586150408
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.001296311477784105
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2916944026947021,
"saved_at_worst": 1.2916944026947021,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.487018426439568,
"worst_tolerance_index": [
3093,
3
],
"actual_at_worst": 0.0007083266973495483,
"saved_at_worst": 0.000708848237991333,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 1.531979432911612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007192505523562431
},
"reported": {
"count": 6272,
"observable_mse": 1.531979432911612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007192505057901144
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002708513307045594
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.2726763998216484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007062527816742659
},
"reported": {
"count": 4092,
"observable_mse": 1.272676763619529e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007062527816742659
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6333547209643545e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 1.6615829736110754e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007192506920546293
},
"reported": {
"count": 896,
"observable_mse": 1.6615824279142544e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007192506920546293
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.447916049061697e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 2.2679167159367353e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.007591899950057268
},
"reported": {
"count": 1284,
"observable_mse": 2.2679158064420335e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.007591899950057268
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.07436711711093e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2836270332336426,
"saved_at_worst": 1.2836270332336426,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5258328609838328,
"worst_tolerance_index": [
1761,
6
],
"actual_at_worst": -0.0004856884479522705,
"saved_at_worst": -0.00048513710498809814,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 2.6668303689802997e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013963649049401283
},
"reported": {
"count": 3136,
"observable_mse": 2.6668309146771207e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013963649049401283
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.442454105776573e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 2.4649105398566462e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013694006018340588
},
"reported": {
"count": 1582,
"observable_mse": 2.46491254074499e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013694005087018013
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0003930625367873124
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 2.6729636374511756e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013963649049401283
},
"reported": {
"count": 448,
"observable_mse": 2.672964910743758e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013963649049401283
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2698981891821986e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 2.9531676773331128e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.014340530149638653
},
"reported": {
"count": 1106,
"observable_mse": 2.9531669497373514e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.014340530149638653
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.254533764833367e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.284090280532837,
"saved_at_worst": 1.284090280532837,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.416902777088917,
"worst_tolerance_index": [
645,
3
],
"actual_at_worst": 0.0010797232389450073,
"saved_at_worst": 0.0010801851749420166,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 9.52184564084746e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026104753836989403
},
"reported": {
"count": 3136,
"observable_mse": 9.521851461613551e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026104751974344254
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0005159002755522424
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 8.958402759162709e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02523515187203884
},
"reported": {
"count": 1001,
"observable_mse": 8.958407124737278e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.025235150009393692
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0005286326718445574
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 9.404146840097383e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026104751974344254
},
"reported": {
"count": 448,
"observable_mse": 9.404151205671951e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026104751974344254
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.32490253115726e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 9.887426858767867e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02660731039941311
},
"reported": {
"count": 1687,
"observable_mse": 9.887434134725481e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02660731039941311
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.204721405824292e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2577426433563232,
"saved_at_worst": 1.2577426433563232,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.3901293702431387,
"worst_tolerance_index": [
1178,
12
],
"actual_at_worst": 0.0006951391696929932,
"saved_at_worst": 0.0006947219371795654,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00047963563702069223,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05295487120747566
},
"reported": {
"count": 3136,
"observable_mse": 0.00047963589895516634,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.052954867482185364
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0005917398364020188
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.00045876813237555325,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.050168175250291824
},
"reported": {
"count": 407,
"observable_mse": 0.0004587684525176883,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.050168171525001526
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006191463366830008
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00048218932352028787,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05295487120747566
},
"reported": {
"count": 448,
"observable_mse": 0.00048218961455859244,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05295487120747566
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002776502956625754
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.000482857518363744,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.053436823189258575
},
"reported": {
"count": 2281,
"observable_mse": 0.0004828577220905572,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.053436823189258575
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.0001943428200573798
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008089635521173477
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00808099564164877
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.650881858950015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.650880221859552e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6343922773059064e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008295925334095955
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6665429939166643e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6665442672092468e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2711741145488733e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008289745077490807
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5941945093800314e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5941997844493017e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.266673141096683e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.027659518644213676
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.701061935629696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.701080853119493e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8917489796876907e-10,
"numeric_tolerance_ratio": 0.00018735732936814417
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025488151237368584
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.793230906827375e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.793223630869761e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.219692932613361e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025727983564138412
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.805288234725595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.805289689917117e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4424899806819776e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01637626811861992
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.188980554114096e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.188985647284426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0769799138515745e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 6.998432908560917e-16
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007534573320299387
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.9181770767318085e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.9181730749551207e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.001776687800884e-11,
"numeric_tolerance_ratio": 3.9941152834059884e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0038363952189683914
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.709371286386158e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.709345820534509e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5443691858507307e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897401075053494e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-13,
"numeric_tolerance_ratio": 4.5440247119973885e-07
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008749740198254585
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0526686057564802e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0526651496766135e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.4490002241757354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.611191146366764e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.080316883302202e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5991054169717245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5991059626685455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.448255872186213e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6771049558883533e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.677106411079876e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4527550979481874e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6066021998994984e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6065991076175123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.092281986027956e-11,
"numeric_tolerance_ratio": 3.087321897422629e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.7853855752036907e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.7853810277301818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-11,
"numeric_tolerance_ratio": 4.539369005564241e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.764368607837241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.7643691535340622e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4473570618693215e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 7.5585703598335385e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 7.558583092759363e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2723308806138392e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.499810635636095e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.4998324635089375e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.181151457204371e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6655649233143777e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6655654690111987e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.447894385869645e-06
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 1.8676328181754798e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 2.704847702261759e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 2.704847702261759e-06
},
"gradients": {
"norms": {
"prediction": 0.00013576024502981454,
"weighted_latent": 2.6831793128829187e-18,
"total": 0.00013576024502981454
},
"prediction_weighted_latent_dot": -9.274898775643064e-24,
"prediction_weighted_latent_cosine": -0.025461669024432854,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "f27c7aa4ae44e1bd2421dfb90baa66d8366a995f651d65613378922892ca067a"
}
],
"final_losses": {
"prediction_mse": 1.8676328181754798e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 2.704847702261759e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 2.704847702261759e-06
},
"training_seconds": 4.040001658722758e-05,
"evaluation_seconds": 0.6968008000403643
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 1,
"arm": "native",
"status": "passed",
"files": {
"seed-1-native/result.json": "891d8fa190de61a1b640fb0ecf66d42cbd9686a51d3e0973c62483357ee349d6",
"seed-1-native/checkpoint.json": "07f14c5b17dac49260c4caf900b11936e5242f4944ae0dbbd3a0ab1c060e6b5b",
"inputs/seed-1-fourier/checkpoint.json": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"inputs/seed-1-fourier/result.json": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"seed-1-native/diagnostics-arrays.npz": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d7fbcbd00af3cc988f25d2a1056bfd8fc13ef058abf92203b194e712bd2a414b",
"parent_checkpoint_sha256": "fe1a6ec9df9e05c0061a9c4078c6b5212da34342eb36126cfaff6220d7c2da4c",
"parent_arrays_sha256": "94ab24b738ad4f007ab5cf33b0c628f1cd3551bdced3cc066af4feab4dcb38d5",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.6193599700927734e-07,
"max_tolerance_ratio": 0.45109829629777515,
"worst_tolerance_index": [
1,
6
],
"actual_at_worst": -0.00023978948593139648,
"saved_at_worst": -0.00024025142192840576,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 1.8676346371648833e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 1.8676328181754798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.815598532142596e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.369931800989434e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 3.369917976669967e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.00013777889110753617
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.291385531425476,
"saved_at_worst": 1.291385531425476,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.42046233387194104,
"worst_tolerance_index": [
87,
6
],
"actual_at_worst": 0.0009859204292297363,
"saved_at_worst": 0.0009863823652267456,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.00011568346235435456,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04658252000808716
},
"reported": {
"count": 784,
"observable_mse": 0.00011568349145818502,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04658252000808716
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8770997512238653e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 0.0001269910135306418,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05011334642767906
},
"reported": {
"count": 580,
"observable_mse": 0.00012699104263447225,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05011334642767906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8738872518211027e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.00012861579307354987,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04658252000808716
},
"reported": {
"count": 112,
"observable_mse": 0.00012861573486588895,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04658252000808716
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.746852525276311e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 2.8653117624344304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"reported": {
"count": 92,
"observable_mse": 2.8653077606577426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004186348523944616
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.001776687800884e-11,
"numeric_tolerance_ratio": 3.990343126672343e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.591147156432271e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205137968063354
},
"reported": {
"count": 98,
"observable_mse": 1.591144609847106e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120511934161186
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0013191059629536056
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2873621926701162e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120512865483761
},
"reported": {
"count": 70,
"observable_mse": 1.28735900943866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041205123998224735
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00032977647986314834
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.774524753272999e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120515659451485
},
"reported": {
"count": 14,
"observable_mse": 1.7745223885867745e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120510071516037
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.003957318410873048
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9266941055539064e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00412051472812891
},
"reported": {
"count": 14,
"observable_mse": 2.9266951969475485e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004120514262467623
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003297764363621434
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6111895092763007e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151817861944437
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151880726218224
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004550374085107517
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.384819915983826e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815179457888007
},
"reported": {
"count": 70,
"observable_mse": 1.3848208254785277e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.0065727623458328764
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4024532902112696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151885382831097
},
"reported": {
"count": 14,
"observable_mse": 1.402454017807031e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003815184347331524
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0030335835414162663
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.9517736038542353e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151845801621675
},
"reported": {
"count": 14,
"observable_mse": 2.9517867005779408e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038151901680976152
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004044776351031832
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6099873391794972e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 98,
"observable_mse": 1.609983155503869e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410440331324935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0029713781524049092
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2653170415433124e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104399122297764
},
"reported": {
"count": 70,
"observable_mse": 1.2653120393224526e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104403778910637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0033015311714488147
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.9126227925880812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"reported": {
"count": 14,
"observable_mse": 1.9126269762637094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004104407504200935
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.007923672718656958
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0307024644571356e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0041044000536203384
},
"reported": {
"count": 14,
"observable_mse": 3.0306960979942232e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00410439632833004
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0026412263323736225
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384033847600222
},
"reported": {
"count": 98,
"observable_mse": 1.5851730495342053e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384037107229233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022661433552030583
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2924613656650763e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374961368739605
},
"reported": {
"count": 72,
"observable_mse": 1.2924631846544798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004374966491013765
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.003563329461384987
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7241960449609905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043840352445840836
},
"reported": {
"count": 14,
"observable_mse": 1.724195317365229e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004384040366858244
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.00356108160832695
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.179249688400887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0044380780309438705
},
"reported": {
"count": 12,
"observable_mse": 3.179239502060227e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004438071511685848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004515324651932781
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5865916793700308e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355667158961296
},
"reported": {
"count": 98,
"observable_mse": 1.5865951354498975e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355666693300009
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0003243745464814044
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.2939838597958442e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043465327471494675
},
"reported": {
"count": 72,
"observable_mse": 1.2939861335325986e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004346531815826893
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006491621714371808
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.7255477359867655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355668555945158
},
"reported": {
"count": 14,
"observable_mse": 1.725551010167692e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004355671815574169
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0022706210151850026
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 3.180122803314589e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004410072695463896
},
"reported": {
"count": 12,
"observable_mse": 3.1801344448467717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00441007548943162
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001938898741991697
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6066007447079755e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 98,
"observable_mse": 1.6065991076175123e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838496565819
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.122274160385132e-09,
"numeric_tolerance_ratio": 0.0036308001127401722
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.2739343219436705e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843153178692
},
"reported": {
"count": 70,
"observable_mse": 1.2739328667521477e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107838962227106
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002970654539643297
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.8545022612670437e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107845015823841
},
"reported": {
"count": 14,
"observable_mse": 1.854493712016847e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107835702598095
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00660145605781308
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.022031523869373e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107843618839979
},
"reported": {
"count": 14,
"observable_mse": 3.0220360713428818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004107840359210968
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0023105088575984434
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0008219711598940194,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130469620227814
},
"reported": {
"count": 98,
"observable_mse": 0.0008219714509323239,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130469620227814
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00026893279647514115
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.0008061013650149107,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130469620227814
},
"reported": {
"count": 84,
"observable_mse": 0.0008061017724685371,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130471110343933
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0010545410041523351
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0009171902202069759,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130469620227814
},
"reported": {
"count": 14,
"observable_mse": 0.0009171901037916541,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.13130471110343933
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0010545410041523351
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897396527579986e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00388339813798666
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834072183817625
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.080395102500916e-09,
"numeric_tolerance_ratio": 0.006540465866677444
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 6.000454959576018e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964682560414076
},
"reported": {
"count": 72,
"observable_mse": 6.000467237754492e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037964771036058664
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006412915697540378
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 7.797699254297186e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038833962753415108
},
"reported": {
"count": 14,
"observable_mse": 7.79771198722301e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0038834111765027046
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.010733069131502397
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 1.6882830095710233e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0043688067235052586
},
"reported": {
"count": 12,
"observable_mse": 1.6882742784218863e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.004368808586150408
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.001296311477784105
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2916944026947021,
"saved_at_worst": 1.2916944026947021,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.487018426439568,
"worst_tolerance_index": [
3093,
3
],
"actual_at_worst": 0.0007083266973495483,
"saved_at_worst": 0.000708848237991333,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.00041410542326048017,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0698724240064621
},
"reported": {
"count": 6272,
"observable_mse": 0.00041410542326048017,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0698724240064621
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.00048100444837473333,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07464633882045746
},
"reported": {
"count": 4092,
"observable_mse": 0.00048100450658239424,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07464633882045746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.553633802648516e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.0004096943885087967,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0698724240064621
},
"reported": {
"count": 896,
"observable_mse": 0.00040969435940496624,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0698724240064621
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.79583909497198e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.00020398187916725874,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0518018938601017
},
"reported": {
"count": 1284,
"observable_mse": 0.00020398193737491965,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.0518018938601017
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.704406502354309e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.1448103189468384,
"saved_at_worst": 1.1448103189468384,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5258328609838328,
"worst_tolerance_index": [
1761,
6
],
"actual_at_worst": -0.0004856884479522705,
"saved_at_worst": -0.00048513710498809814,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0008384898537769914,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09975315630435944
},
"reported": {
"count": 3136,
"observable_mse": 0.0008384898537769914,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09975315630435944
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.0009020490688271821,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10331261903047562
},
"reported": {
"count": 1582,
"observable_mse": 0.0009020491852425039,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10331261903047562
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.0001067829724933912
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0008328369585797191,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09975315630435944
},
"reported": {
"count": 448,
"observable_mse": 0.0008328371332027018,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09975315630435944
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016119782896502883
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.0007498656050302088,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09442884474992752
},
"reported": {
"count": 1106,
"observable_mse": 0.0007498656632378697,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09442883729934692
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0007134600738267936
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2048335075378418,
"saved_at_worst": 1.2048335075378418,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.513429641723633e-07,
"max_tolerance_ratio": 0.416902777088917,
"worst_tolerance_index": [
645,
3
],
"actual_at_worst": 0.0010797232389450073,
"saved_at_worst": 0.0010801851749420166,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.006654600612819195,
"exact_state_accuracy": 0.9754464285714286,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.9754464285714286
},
"normalized_latent_rmse": 0.2049500048160553
},
"reported": {
"count": 3136,
"observable_mse": 0.006654600612819195,
"exact_state_accuracy": 0.9754464030265808,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.9754464030265808
},
"normalized_latent_rmse": 0.2049500048160553
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.5544847792602354e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00838522706180811,
"exact_state_accuracy": 0.964035964035964,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.964035964035964
},
"normalized_latent_rmse": 0.22751471400260925
},
"reported": {
"count": 1001,
"observable_mse": 0.00838522519916296,
"exact_state_accuracy": 0.9640359282493591,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.9640359282493591
},
"normalized_latent_rmse": 0.22751472890377045
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.578660490166641e-08,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0010131206602330654
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00667498679831624,
"exact_state_accuracy": 0.9754464285714286,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.9754464285714286
},
"normalized_latent_rmse": 0.2049499899148941
},
"reported": {
"count": 448,
"observable_mse": 0.006674987263977528,
"exact_state_accuracy": 0.9754464626312256,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.9754464626312256
},
"normalized_latent_rmse": 0.2049499899148941
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.405979698278827e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002792573571037702
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.005622301250696182,
"exact_state_accuracy": 0.98221695317131,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.98221695317131
},
"normalized_latent_rmse": 0.1903003305196762
},
"reported": {
"count": 1687,
"observable_mse": 0.005622302182018757,
"exact_state_accuracy": 0.982217013835907,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 0.982217013835907
},
"normalized_latent_rmse": 0.1903003305196762
},
"differences": [],
"max_errors": {
"accuracy_abs": 6.066459701248306e-08,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005961493791148332
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.1234519481658936,
"saved_at_worst": 1.1234519481658936,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.289912223815918e-07,
"max_tolerance_ratio": 0.3310985707951908,
"worst_tolerance_index": [
1621,
9
],
"actual_at_worst": -0.00170096755027771,
"saved_at_worst": -0.00170135498046875,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.01799476332962513,
"exact_state_accuracy": 0.9585459183673469,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9862882653061225,
"h": 0.9674744897959183
},
"normalized_latent_rmse": 0.2951428294181824
},
"reported": {
"count": 3136,
"observable_mse": 0.01799476519227028,
"exact_state_accuracy": 0.9585459232330322,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9862882494926453,
"h": 0.9674744606018066
},
"normalized_latent_rmse": 0.29514279961586
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.9194111683672475e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0009766680526367667
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.02545979805290699,
"exact_state_accuracy": 0.9238329238329238,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9901719901719902,
"h": 0.9336609336609336
},
"normalized_latent_rmse": 0.33765652775764465
},
"reported": {
"count": 407,
"observable_mse": 0.02545979991555214,
"exact_state_accuracy": 0.923832893371582,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9901719689369202,
"h": 0.9336609244346619
},
"normalized_latent_rmse": 0.33765649795532227
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.0461341782306306e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0008572347291931027
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.018050488084554672,
"exact_state_accuracy": 0.9620535714285714,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9866071428571429,
"h": 0.96875
},
"normalized_latent_rmse": 0.2951428294181824
},
"reported": {
"count": 448,
"observable_mse": 0.01805048994719982,
"exact_state_accuracy": 0.9620535969734192,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9866071939468384,
"h": 0.9687500596046448
},
"normalized_latent_rmse": 0.2951428294181824
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006640330178678031
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.016651831567287445,
"exact_state_accuracy": 0.964050854888207,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9855326611135466,
"h": 0.9732573432704954
},
"normalized_latent_rmse": 0.2868955135345459
},
"reported": {
"count": 2281,
"observable_mse": 0.016651831567287445,
"exact_state_accuracy": 0.9640508890151978,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9855326414108276,
"h": 0.9732573628425598
},
"normalized_latent_rmse": 0.2868955135345459
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.4126990788863054e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008089635521173477
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6376299754483625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00808099564164877
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.650881858950015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.650880221859552e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6343922773059064e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008295925334095955
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6665429939166643e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6665442672092468e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2711741145488733e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008289745077490807
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5941945093800314e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5941997844493017e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.266673141096683e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.027659518644213676
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.701061935629696e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.701080853119493e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8917489796876907e-10,
"numeric_tolerance_ratio": 0.00018735732936814417
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025488151237368584
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.793230906827375e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.793223630869761e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.219692932613361e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.025727983564138412
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.805288234725595e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.805289689917117e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4424899806819776e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.01637626811861992
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.188980554114096e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.188985647284426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0769799138515745e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.2242463231086731
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.008719741366803646,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.008719741366803646,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.22663171589374542
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.009586147964000702,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.009586147964000702,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0822481019422412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0822448277613148e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.267377446594027e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0038363952189683914
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 8.709371286386158e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 8.709345820534509e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5443691858507307e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.5897401075053494e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.5897405622527e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-13,
"numeric_tolerance_ratio": 4.5440247119973885e-07
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008749740198254585
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.0526686057564802e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0526651496766135e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.4490002241757354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.611191146366764e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6111920558614656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.080316883302202e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5991054169717245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5991059626685455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.448255872186213e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6771049558883533e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.677106411079876e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4527550979481874e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.001706356881186366,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0017063572304323316,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.0002983387219492074
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0008994330419227481,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0008994329255074263,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010680860428480968
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.764368607837241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.7643691535340622e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4473570618693215e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 7.5585703598335385e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 7.558583092759363e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2723308806138392e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.499810635636095e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.4998324635089375e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.181151457204371e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.6655649233143777e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.6655654690111987e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.447894385869645e-06
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 70,
"error_count": 28,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 70
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 177,
"error_count": 19,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 177
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 58,
"error_count": 40,
"per_field_correct_counts": {
"x": 98,
"y": 62,
"h": 86
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 55,
"error_count": 43,
"per_field_correct_counts": {
"x": 98,
"y": 91,
"h": 55
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3006,
"accuracy": 0.9585459183673469
},
"numpy": {
"cases": 3136,
"correct": 3006,
"accuracy": 0.9585459183673469
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0008061017724685371,
"latent_mse": 0.01724092662334442,
"scaled_prediction": 0.00011674577399389818,
"scaled_weighted_latent": 0.000249696196988225,
"total": 0.00036644196370616555
},
"gradients": {
"norms": {
"prediction": 0.0013187306467443705,
"weighted_latent": 0.0013792344834655523,
"total": 0.0023915665224194527
},
"prediction_weighted_latent_dot": 1.0391254363639746e-06,
"prediction_weighted_latent_cosine": 0.571312557453286,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "07f14c5b17dac49260c4caf900b11936e5242f4944ae0dbbd3a0ab1c060e6b5b"
}
],
"final_losses": {
"prediction_mse": 0.0008061017724685371,
"latent_mse": 0.01724092662334442,
"scaled_prediction": 0.00011674577399389818,
"scaled_weighted_latent": 0.000249696196988225,
"total": 0.00036644196370616555
},
"training_seconds": 3.9599952287971973e-05,
"evaluation_seconds": 0.53689819999272
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 2,
"arm": "latent_only",
"status": "passed",
"files": {
"seed-2-latent_only/result.json": "ff7cdb5c62fd466904afbdbd3a4e9642acd8ccdffccc3e52e4a1092c51bda229",
"seed-2-latent_only/checkpoint.json": "98a0907e90e006e7eceee8a2a878b8fafbfaa046337219367a17ac48357802cd",
"inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"seed-2-latent_only/diagnostics-arrays.npz": "f750723b6f773a15bc25f177a7aff420f0aba851bd55805f2ab9a5f07314e702"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.172325134277344e-07,
"max_tolerance_ratio": 0.38184041187270573,
"worst_tolerance_index": [
90,
15
],
"actual_at_worst": -0.0009272992610931396,
"saved_at_worst": -0.0009268820285797119,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.4284559913212433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.428454718028661e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2702079400172872e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926299571990967,
"saved_at_worst": 1.2926299571990967,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5302791797917392,
"worst_tolerance_index": [
545,
1
],
"actual_at_worst": -0.0012408345937728882,
"saved_at_worst": -0.0012402385473251343,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.365978252782952e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033088121563196182
},
"reported": {
"count": 784,
"observable_mse": 2.36597716138931e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033088119234889746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017494472458728073
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.978542422875762e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003269587177783251
},
"reported": {
"count": 580,
"observable_mse": 1.978542968572583e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003269587177783251
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4461926843963654e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.7823572963825427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003308812854811549
},
"reported": {
"count": 112,
"observable_mse": 2.782348747132346e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003308812854811549
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.549250196665525e-11,
"numeric_tolerance_ratio": 8.525529201174e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 4.30161198892165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"reported": {
"count": 92,
"observable_mse": 4.301610169932246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.811198334370962e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4489330826327205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 98,
"observable_mse": 2.448930717946496e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749081823974848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333975094181
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0924091586493887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925644695759
},
"reported": {
"count": 70,
"observable_mse": 2.0924004275002517e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749084152281284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333754161296
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1905292669543996e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 14,
"observable_mse": 2.1905329049332067e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474907483905554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.792795956134796e-08,
"numeric_tolerance_ratio": 0.01330469955564529
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.489958519116044e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474924247711897
},
"reported": {
"count": 14,
"observable_mse": 4.489981074584648e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00347490725107491
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6996636986732483e-08,
"numeric_tolerance_ratio": 0.012613546549922739
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4650891646160744e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355574123561382
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017459385640157722
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0494757336564362e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 70,
"observable_mse": 2.049479189736303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355571795254946
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00034918772499636073
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.5227938749594614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335555549710989
},
"reported": {
"count": 14,
"observable_mse": 2.5227842343156226e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355604391545057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.0036664702162613125
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.485453246161342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 14,
"observable_mse": 4.485439421841875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355567138642073
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006983754743791358
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4724424292799085e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488371381536126
},
"reported": {
"count": 98,
"observable_mse": 2.4724389732000418e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883692860603333
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0954757928848267e-09,
"numeric_tolerance_ratio": 0.0015535427214692392
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9836363208014518e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488370683044195
},
"reported": {
"count": 70,
"observable_mse": 1.983637412195094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012083110264445502
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.7921978471567854e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883730113506317
},
"reported": {
"count": 14,
"observable_mse": 2.792183840938378e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488372080028057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006904632887422107
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.596718281391077e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883723128587008
},
"reported": {
"count": 14,
"observable_mse": 4.596703365677968e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0024166220528891004
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4542685423512012e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660863498225808
},
"reported": {
"count": 98,
"observable_mse": 2.4542630853829905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660864196717739
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0005113087436513964
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9375931515241973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036534646060317755
},
"reported": {
"count": 72,
"observable_mse": 1.9375880583538674e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00365346553735435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006821144214759868
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0936709663365036e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608625669032335
},
"reported": {
"count": 14,
"observable_mse": 3.093665145570412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608658265322447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002386107185697774
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8083511501317844e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704948117956519
},
"reported": {
"count": 12,
"observable_mse": 4.808346784557216e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704944159835577
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.958120942115784e-09,
"numeric_tolerance_ratio": 0.002888097095437761
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.454017158015631e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00366382603533566
},
"reported": {
"count": 98,
"observable_mse": 2.4540126105421223e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638197489082813
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004600783305236997
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9373417671886273e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656336572021246
},
"reported": {
"count": 72,
"observable_mse": 1.937336492119357e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656331915408373
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0034098562497762368
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.09398710669484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638290621340275
},
"reported": {
"count": 14,
"observable_mse": 3.0939725547796115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003663820680230856
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.0061343773221983556
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8074376536533237e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003708441974595189
},
"reported": {
"count": 12,
"observable_mse": 4.8074492951855063e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00370843056589365
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1408701539039612e-08,
"numeric_tolerance_ratio": 0.008322398019379602
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4505963665433228e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 98,
"observable_mse": 2.4505910914740525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0662389943026938e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 70,
"observable_mse": 2.066234446829185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.32611782848835e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416656516492367
},
"reported": {
"count": 14,
"observable_mse": 2.3261030946741812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416670486330986
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.010412299112111879
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.4968615839025006e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416659077629447
},
"reported": {
"count": 14,
"observable_mse": 4.4968630390940234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416667925193906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006594457363167671
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7606145522440784e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.6749262360681314e-07
},
"reported": {
"count": 98,
"observable_mse": 2.760614916041959e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.663332452357281e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1593783710850403e-09,
"numeric_tolerance_ratio": 0.001159347493807002
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.428484003758058e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.6760042715068266e-07
},
"reported": {
"count": 84,
"observable_mse": 2.428484003758058e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.656196045336401e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.980822617042577e-09,
"numeric_tolerance_ratio": 0.001980770003908066
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.7533987526549026e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.668449496923131e-07
},
"reported": {
"count": 14,
"observable_mse": 4.75340239063371e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.7057549800701963e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.730548314706539e-09,
"numeric_tolerance_ratio": 0.003730447377940832
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218651813280303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703649388626218
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703637048602104
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.009004926261465663
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.265882656298345e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003696256782859564
},
"reported": {
"count": 72,
"observable_mse": 1.2658831110456958e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036962456069886684
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.008159806136714668
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4861635463603307e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703654045239091
},
"reported": {
"count": 14,
"observable_mse": 1.4861591807857621e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037036393769085407
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4668330550193787e-08,
"numeric_tolerance_ratio": 0.010703967133658529
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.2827452994533814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037476930301636457
},
"reported": {
"count": 12,
"observable_mse": 2.2827287466498092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00374767673201859
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-08,
"numeric_tolerance_ratio": 0.01185519951732077
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29456627368927,
"saved_at_worst": 1.29456627368927,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.48837328246516154,
"worst_tolerance_index": [
1481,
14
],
"actual_at_worst": 6.841123104095459e-05,
"saved_at_worst": 6.890296936035156e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.3900793166831136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006172883789986372
},
"reported": {
"count": 6272,
"observable_mse": 2.3900787709862925e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006172883789986372
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.44395672523841e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 2.0221228623995557e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006107353139668703
},
"reported": {
"count": 4092,
"observable_mse": 2.0221217710059136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006107353139668703
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0891911649600609e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.621443672978785e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006172882858663797
},
"reported": {
"count": 896,
"observable_mse": 2.6214440367766656e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006172883324325085
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.000287927191441092
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.401274807401933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006377234123647213
},
"reported": {
"count": 1284,
"observable_mse": 3.401275898795575e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.006377233657985926
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028433452012249446
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2953126430511475,
"saved_at_worst": 1.2953126430511475,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4286914392087942,
"worst_tolerance_index": [
2524,
1
],
"actual_at_worst": 0.0011226087808609009,
"saved_at_worst": 0.001123085618019104,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.592955908970907e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011947340331971645
},
"reported": {
"count": 3136,
"observable_mse": 3.592955908970907e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011947340331971645
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 3.2867963454918936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011877006851136684
},
"reported": {
"count": 1582,
"observable_mse": 3.286795981694013e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01187700591981411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004257084255629218
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.645531978690997e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01194733940064907
},
"reported": {
"count": 448,
"observable_mse": 3.645533070084639e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011947340331971645
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00042434416222123303
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 4.0095834265230224e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012047233060002327
},
"reported": {
"count": 1106,
"observable_mse": 4.009585245512426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012047232128679752
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00042242153989206836
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2925742864608765,
"saved_at_worst": 1.2925742864608765,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4916956737725486,
"worst_tolerance_index": [
36,
11
],
"actual_at_worst": 0.0009863413870334625,
"saved_at_worst": 0.0009858012199401855,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00010225340520264581,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022724181413650513
},
"reported": {
"count": 3136,
"observable_mse": 0.00010225344158243388,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022724179551005363
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0005691953701475557
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 9.7694355645217e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0224163755774498
},
"reported": {
"count": 1001,
"observable_mse": 9.76944065769203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0224163755774498
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.04389430384333e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00010278232366545126,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02272418513894081
},
"reported": {
"count": 448,
"observable_mse": 0.00010278236004523933,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02272418513894081
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.600967216199065e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00010481807839823887,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022904863581061363
},
"reported": {
"count": 1687,
"observable_mse": 0.0001048181438818574,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022904863581061363
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.548361852765083e-11,
"numeric_tolerance_ratio": 6.48043513453026e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2814282178878784,
"saved_at_worst": 1.2814282178878784,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 4.917383193969727e-07,
"max_tolerance_ratio": 0.417693559955126,
"worst_tolerance_index": [
2629,
11
],
"actual_at_worst": 0.0007911957800388336,
"saved_at_worst": 0.0007916465401649475,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00046192927402444184,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04700247570872307
},
"reported": {
"count": 3136,
"observable_mse": 0.0004619293613359332,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04700247570872307
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.345639542633262e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.00044614719809032977,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04471798986196518
},
"reported": {
"count": 407,
"observable_mse": 0.0004461473145056516,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04471799358725548
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006808163191366691
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0004608675662893802,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04700247570872307
},
"reported": {
"count": 448,
"observable_mse": 0.0004608677118085325,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04700247570872307
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.000139108108708231
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0004649537440855056,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04739852622151375
},
"reported": {
"count": 2281,
"observable_mse": 0.00046495391870848835,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04739852622151375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016686455009440977
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006795178167521954
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4145125280483626e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4145105271600187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9960688147343306e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006793184205889702
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4182987544918433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4183040295611136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.262343324217096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007076622452586889
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4742850655457005e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.474279244779609e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.806399437731063e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00707575399428606
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4824970751069486e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4825025320751593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.44345482026303e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02292351797223091
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.174026930239052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.173992005642503e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00034710295461388345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02197711542248726
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.503961438080296e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.5039843573467806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.2919266484677792e-10,
"numeric_tolerance_ratio": 0.0002279380971257544
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.022106481716036797
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.032768189674243e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.03276673448272e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4450289711579617e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.013866852968931198
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.329649189254269e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.3296459150733426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.26331524211703e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 5.192282515054103e-07
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7606429284787737e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.760647657851223e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.716352262392452e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0063375686295330524
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8662645490840077e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8662767363130115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2187229003757238e-10,
"numeric_tolerance_ratio": 0.00012152396871314546
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.003950412385165691
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.004109410336241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0041121388203464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-11,
"numeric_tolerance_ratio": 2.7230268541460572e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218641808838584e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4531253780259127e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007395219057798386
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7916874387301505e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7916930775973015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.638867150992155e-11,
"numeric_tolerance_ratio": 5.623168989051261e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.465091529302299e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.266129628595387e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4603585188742727e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.460366340528708e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.821654435247183e-11,
"numeric_tolerance_ratio": 7.802457531363612e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.408349246252328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4083446987788193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-11,
"numeric_tolerance_ratio": 4.536547937688154e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.450601641612593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4506001864210702e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4516341479231694e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8193089747219346e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8193100661155768e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883253156100062e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.6724419512902386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.672434493433684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.457856554538012e-11,
"numeric_tolerance_ratio": 7.437979042782642e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.5223119589791168e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.5223105947370641e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3642420526593924e-11,
"numeric_tolerance_ratio": 1.3621684092581624e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5361989426310174e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5361960322479717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.90591898445941e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.4043375964974985e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.404349238029681e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011590483644883742
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0006819527479819953,
"latent_mse": 0.01885349676012993,
"scaled_prediction": 9.876557305688038e-05,
"scaled_weighted_latent": 0.0002730506530497223,
"total": 0.0002730506530497223
},
"gradients": {
"norms": {
"prediction": 0.0009655424510128796,
"weighted_latent": 0.001503756851889193,
"total": 0.001503756851889193
},
"prediction_weighted_latent_dot": 9.367491315970256e-07,
"prediction_weighted_latent_cosine": 0.6451702116278689,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0007797321886755526,
"latent_mse": 0.017864245921373367,
"scaled_prediction": 0.00011292673298157752,
"scaled_weighted_latent": 0.00025872356491163373,
"total": 0.00025872356491163373
},
"gradients": {
"norms": {
"prediction": 0.0017306143417954445,
"weighted_latent": 0.0014648055657744408,
"total": 0.0014648055657744408
},
"prediction_weighted_latent_dot": 1.1071567769249668e-06,
"prediction_weighted_latent_cosine": 0.43674590614962144,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.03140963986515999
},
{
"step": 100,
"losses": {
"prediction_mse": 0.00012653719750232995,
"latent_mse": 0.00035402210778556764,
"scaled_prediction": 1.8326078134123236e-05,
"scaled_weighted_latent": 5.127216809341917e-06,
"total": 5.127216809341917e-06
},
"gradients": {
"norms": {
"prediction": 0.0017391019500792027,
"weighted_latent": 0.00022246266598813236,
"total": 0.00022246266598813236
},
"prediction_weighted_latent_dot": 1.5913174422621523e-07,
"prediction_weighted_latent_cosine": 0.41131509061071686,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.001169891795143485
},
{
"step": 200,
"losses": {
"prediction_mse": 2.367920933465939e-05,
"latent_mse": 3.444951914843841e-07,
"scaled_prediction": 3.4294027955183992e-06,
"scaled_weighted_latent": 4.989241020325608e-09,
"total": 4.989241020325608e-09
},
"gradients": {
"norms": {
"prediction": 7.438002648996189e-05,
"weighted_latent": 6.940542789379833e-06,
"total": 6.940542789379833e-06
},
"prediction_weighted_latent_dot": -3.408873461285822e-10,
"prediction_weighted_latent_cosine": -0.6603301324194677,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 6.886219489388168e-05
},
{
"step": 300,
"losses": {
"prediction_mse": 2.4286680854856968e-05,
"latent_mse": 2.895395728730321e-12,
"scaled_prediction": 3.5173814012523508e-06,
"scaled_weighted_latent": 4.193331811885685e-14,
"total": 4.193331811885685e-14
},
"gradients": {
"norms": {
"prediction": 0.00010050291166407987,
"weighted_latent": 2.00909866521215e-08,
"total": 2.00909866521215e-08
},
"prediction_weighted_latent_dot": 1.4684928620334325e-12,
"prediction_weighted_latent_cosine": 0.7272637331027412,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.0570345665428249e-07
},
{
"step": 400,
"losses": {
"prediction_mse": 2.4284863684442826e-05,
"latent_mse": 7.688360852912274e-14,
"scaled_prediction": 3.517118329909863e-06,
"scaled_weighted_latent": 1.1134867484500458e-15,
"total": 1.1134867484500458e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040963388746604,
"weighted_latent": 3.113640012486485e-09,
"total": 3.113640012486485e-09
},
"prediction_weighted_latent_dot": 2.099499637727223e-13,
"prediction_weighted_latent_cosine": 0.6715402079953351,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.5189179691788013e-07
},
{
"step": 500,
"losses": {
"prediction_mse": 2.428483458061237e-05,
"latent_mse": 6.981860158376471e-14,
"scaled_prediction": 3.5171140098100295e-06,
"scaled_weighted_latent": 1.0111660154546735e-15,
"total": 1.0111660154546735e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040985944215208,
"weighted_latent": 3.031748407877899e-09,
"total": 3.031748407877899e-09
},
"prediction_weighted_latent_dot": 2.0976327771114744e-13,
"prediction_weighted_latent_cosine": 0.6890646067087806,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.7064159862911765e-07
},
{
"step": 600,
"losses": {
"prediction_mse": 2.4284812752739526e-05,
"latent_mse": 6.92448350178218e-14,
"scaled_prediction": 3.5171108265785733e-06,
"scaled_weighted_latent": 1.0028563046047536e-15,
"total": 1.0028563046047536e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040901543106884,
"weighted_latent": 3.0500808545497193e-09,
"total": 3.0500808545497193e-09
},
"prediction_weighted_latent_dot": 2.077947866942556e-13,
"prediction_weighted_latent_cosine": 0.6785011477866526,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.9513116455982527e-07
},
{
"step": 700,
"losses": {
"prediction_mse": 2.4284869141411036e-05,
"latent_mse": 7.430133033359612e-14,
"scaled_prediction": 3.5171190120308893e-06,
"scaled_weighted_latent": 1.0760882321255187e-15,
"total": 1.0760882321255187e-15
},
"gradients": {
"norms": {
"prediction": 0.00010041041969088838,
"weighted_latent": 3.114990265729034e-09,
"total": 3.114990265729034e-09
},
"prediction_weighted_latent_dot": 2.1831597944374503e-13,
"prediction_weighted_latent_cosine": 0.6979913627514204,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.2110079928315827e-07
},
{
"step": 800,
"losses": {
"prediction_mse": 2.4284858227474615e-05,
"latent_mse": 7.316722775612197e-14,
"scaled_prediction": 3.517117420415161e-06,
"scaled_weighted_latent": 1.0596633103661921e-15,
"total": 1.0596633103661921e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040940105682239,
"weighted_latent": 3.103554968575395e-09,
"total": 3.103554968575395e-09
},
"prediction_weighted_latent_dot": 2.1546756349619112e-13,
"prediction_weighted_latent_cosine": 0.6914297826108758,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.905540270603524e-07
},
{
"step": 900,
"losses": {
"prediction_mse": 2.4284858227474615e-05,
"latent_mse": 7.161070645972031e-14,
"scaled_prediction": 3.517117420415161e-06,
"scaled_weighted_latent": 1.037120587266203e-15,
"total": 1.037120587266203e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040773486252874,
"weighted_latent": 3.1429283620099113e-09,
"total": 3.1429283620099113e-09
},
"prediction_weighted_latent_dot": 1.9667293238868205e-13,
"prediction_weighted_latent_cosine": 0.6232222376147312,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.7591416912619025e-07
},
{
"step": 1000,
"losses": {
"prediction_mse": 2.428484003758058e-05,
"latent_mse": 7.05537787481364e-14,
"scaled_prediction": 3.5171149193047313e-06,
"scaled_weighted_latent": 1.0218133254807785e-15,
"total": 1.0218133254807785e-15
},
"gradients": {
"norms": {
"prediction": 0.00010040798224508762,
"weighted_latent": 3.095038891842705e-09,
"total": 3.095038891842705e-09
},
"prediction_weighted_latent_dot": 2.1120832947169044e-13,
"prediction_weighted_latent_cosine": 0.6796364944217402,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.087874287961313e-07
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "aa58dbb3d60e93a7ba96625c370c99c56f57dd6b174a7e40abad84e98e9cac39"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "7efd99d08941e8b46c20b7d4b7394b1024c87e7f310403a25de38d6795156ff5"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "2ae06d59e09d369623d533ad194c9d63fe8bf0a4b141f7950b0faedb58402076"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "aa9e656b7f28eef673d21d5012f224692c51cbe3b8d670cd17e29f5bab4785f1"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "01b2901ad9c01862b2a992897fd6c3001d6f8f1acb4236e79604e870bc0ede41"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "733983eb538fef59a951b02b83d9c228b3cd8533a7ec8123a8ba1b4eaa5a6c95"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "30ff275a169fa2a6ae7f10ca9cf651bc0ae107e456350f4113c60826618f0531"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "f947f2ca56f1ae84ba18478db8296a21837fc5cfa17f382023155b6fbf5b4707"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "29750935156390726ab063c1f8e3ec7f83a25f0f6c6eca066f62b17a260af58e"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "69243f60cb26df352c8e97589ed6e917f9ce98b40b8b7ec29889763d66ed5b64"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "e7ab4b9fcee645d495ed002c4e7bbc41e216974e8ada148404b161dd28dbb827"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "98a0907e90e006e7eceee8a2a878b8fafbfaa046337219367a17ac48357802cd"
}
],
"final_losses": {
"prediction_mse": 2.428484003758058e-05,
"latent_mse": 7.05537787481364e-14,
"scaled_prediction": 3.5171149193047313e-06,
"scaled_weighted_latent": 1.0218133254807785e-15,
"total": 1.0218133254807785e-15
},
"training_seconds": 2.756893600046169,
"evaluation_seconds": 0.5482100999797694
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 2,
"arm": "refit",
"status": "passed",
"files": {
"seed-2-refit/result.json": "c9ba2578682b8785b6a32a4b326b907148f9c1c11ea4fbfc188cf90a838fd908",
"seed-2-refit/checkpoint.json": "bda2f8457e2047e74b2f4197dcf8e137d5b22a58fb906b015a5af31e57ab3bd4",
"inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"seed-2-refit/diagnostics-arrays.npz": "a1f9bbdf9024c8215f4cb7b88b9dc9ad2907426b95ab3e1cf1fbc0609c935279"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.172325134277344e-07,
"max_tolerance_ratio": 0.38184041187270573,
"worst_tolerance_index": [
90,
15
],
"actual_at_worst": -0.0009272992610931396,
"saved_at_worst": -0.0009268820285797119,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.4284559913212433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.428454718028661e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2702079400172872e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926299571990967,
"saved_at_worst": 1.2926299571990967,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5302791797917392,
"worst_tolerance_index": [
545,
1
],
"actual_at_worst": -0.0012408345937728882,
"saved_at_worst": -0.0012402385473251343,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.365975160500966e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033088121563196182
},
"reported": {
"count": 784,
"observable_mse": 2.3659733415115625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033088119234889746
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017494472458728073
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.9785386029980145e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003269587177783251
},
"reported": {
"count": 580,
"observable_mse": 1.9785384210990742e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003269587177783251
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-12,
"numeric_tolerance_ratio": 1.8153975697046262e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.7823556592920795e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003308812854811549
},
"reported": {
"count": 112,
"observable_mse": 2.7823460186482407e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003308812854811549
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.640643838793039e-11,
"numeric_tolerance_ratio": 9.613894657269682e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 4.30161198892165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"reported": {
"count": 92,
"observable_mse": 4.301610169932246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.811198334370962e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4489330826327205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 98,
"observable_mse": 2.448930717946496e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749081823974848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333975094181
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0924091586493887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925644695759
},
"reported": {
"count": 70,
"observable_mse": 2.0924004275002517e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749084152281284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333754161296
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1905292669543996e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 14,
"observable_mse": 2.1905329049332067e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474907483905554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.792795956134796e-08,
"numeric_tolerance_ratio": 0.01330469955564529
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.489958519116044e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474924247711897
},
"reported": {
"count": 14,
"observable_mse": 4.489981074584648e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00347490725107491
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6996636986732483e-08,
"numeric_tolerance_ratio": 0.012613546549922739
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4650891646160744e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355574123561382
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017459385640157722
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0494757336564362e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 70,
"observable_mse": 2.049479189736303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355571795254946
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00034918772499636073
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.5227938749594614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335555549710989
},
"reported": {
"count": 14,
"observable_mse": 2.5227842343156226e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355604391545057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.0036664702162613125
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.485453246161342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 14,
"observable_mse": 4.485439421841875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355567138642073
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006983754743791358
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4724424292799085e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488371381536126
},
"reported": {
"count": 98,
"observable_mse": 2.4724389732000418e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883692860603333
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0954757928848267e-09,
"numeric_tolerance_ratio": 0.0015535427214692392
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9836363208014518e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488370683044195
},
"reported": {
"count": 70,
"observable_mse": 1.983637412195094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012083110264445502
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.7921978471567854e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883730113506317
},
"reported": {
"count": 14,
"observable_mse": 2.792183840938378e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488372080028057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006904632887422107
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.596718281391077e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883723128587008
},
"reported": {
"count": 14,
"observable_mse": 4.596703365677968e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0024166220528891004
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4542685423512012e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660863498225808
},
"reported": {
"count": 98,
"observable_mse": 2.4542630853829905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660864196717739
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0005113087436513964
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9375931515241973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036534646060317755
},
"reported": {
"count": 72,
"observable_mse": 1.9375880583538674e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00365346553735435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006821144214759868
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0936709663365036e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608625669032335
},
"reported": {
"count": 14,
"observable_mse": 3.093665145570412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608658265322447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002386107185697774
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8083511501317844e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704948117956519
},
"reported": {
"count": 12,
"observable_mse": 4.808346784557216e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704944159835577
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.958120942115784e-09,
"numeric_tolerance_ratio": 0.002888097095437761
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.454017158015631e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00366382603533566
},
"reported": {
"count": 98,
"observable_mse": 2.4540126105421223e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638197489082813
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004600783305236997
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9373417671886273e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656336572021246
},
"reported": {
"count": 72,
"observable_mse": 1.937336492119357e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656331915408373
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0034098562497762368
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.09398710669484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638290621340275
},
"reported": {
"count": 14,
"observable_mse": 3.0939725547796115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003663820680230856
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.0061343773221983556
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8074376536533237e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003708441974595189
},
"reported": {
"count": 12,
"observable_mse": 4.8074492951855063e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00370843056589365
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1408701539039612e-08,
"numeric_tolerance_ratio": 0.008322398019379602
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4505963665433228e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 98,
"observable_mse": 2.4505910914740525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0662389943026938e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 70,
"observable_mse": 2.066234446829185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.32611782848835e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416656516492367
},
"reported": {
"count": 14,
"observable_mse": 2.3261030946741812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416670486330986
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.010412299112111879
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.4968615839025006e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416659077629447
},
"reported": {
"count": 14,
"observable_mse": 4.4968630390940234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416667925193906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006594457363167671
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605889044934884e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.632583573764565e-05
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.428456355119124e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"reported": {
"count": 84,
"observable_mse": 2.428454718028661e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6331244943079406e-05
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.7533838369417936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360026644082518e-16
},
"reported": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360023997104558e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6207679087541517e-06
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218651813280303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703649388626218
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703637048602104
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.009004926261465663
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.265882656298345e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003696256782859564
},
"reported": {
"count": 72,
"observable_mse": 1.2658831110456958e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036962456069886684
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.008159806136714668
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4861635463603307e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703654045239091
},
"reported": {
"count": 14,
"observable_mse": 1.4861591807857621e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037036393769085407
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4668330550193787e-08,
"numeric_tolerance_ratio": 0.010703967133658529
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.2827452994533814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037476930301636457
},
"reported": {
"count": 12,
"observable_mse": 2.2827287466498092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00374767673201859
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-08,
"numeric_tolerance_ratio": 0.01185519951732077
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29456627368927,
"saved_at_worst": 1.29456627368927,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.48837328246516154,
"worst_tolerance_index": [
1481,
14
],
"actual_at_worst": 6.841123104095459e-05,
"saved_at_worst": 6.890296936035156e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.378944009251427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006171076558530331
},
"reported": {
"count": 6272,
"observable_mse": 2.3789432816556655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006171076092869043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002879593693292197
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 2.011634933296591e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006105360575020313
},
"reported": {
"count": 4092,
"observable_mse": 2.0116334781050682e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006105361040681601
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028913433615768966
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.6096893634530716e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006171076092869043
},
"reported": {
"count": 896,
"observable_mse": 2.6096897272509523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006171076092869043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6285095230641403e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.3885084121720865e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0063759880140423775
},
"reported": {
"count": 1284,
"observable_mse": 3.388509503565729e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.006375988479703665
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002843561400185876
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2946816682815552,
"saved_at_worst": 1.2946816682815552,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4679874330909149,
"worst_tolerance_index": [
861,
7
],
"actual_at_worst": -0.0005080029368400574,
"saved_at_worst": -0.0005075111985206604,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.5174623917555436e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01194077916443348
},
"reported": {
"count": 3136,
"observable_mse": 3.5174627555534244e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01194077916443348
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.625227205416242e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 3.2099360396387056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011869578622281551
},
"reported": {
"count": 1582,
"observable_mse": 3.209937131032348e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011869577690958977
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004258530218443556
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.567524254322052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01194077916443348
},
"reported": {
"count": 448,
"observable_mse": 3.5675260733114555e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01194077916443348
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.812523179843284e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 3.937062501790933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012041891925036907
},
"reported": {
"count": 1106,
"observable_mse": 3.937062865588814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012041891925036907
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6237120250423318e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2910401821136475,
"saved_at_worst": 1.2910401821136475,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.43953577764266183,
"worst_tolerance_index": [
2187,
4
],
"actual_at_worst": -0.000593923032283783,
"saved_at_worst": -0.0005943886935710907,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 9.813705401029438e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022713838145136833
},
"reported": {
"count": 3136,
"observable_mse": 9.813706128625199e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.022713836282491684
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0005693753349948253
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 9.348728053737432e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0224068034440279
},
"reported": {
"count": 1001,
"observable_mse": 9.348732419312e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0224068034440279
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.32513999204833e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 9.860369755188003e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02271384373307228
},
"reported": {
"count": 448,
"observable_mse": 9.86036320682615e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02271384373307228
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.548361852765083e-11,
"numeric_tolerance_ratio": 6.484423085950879e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00010077212209580466,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02289407141506672
},
"reported": {
"count": 1687,
"observable_mse": 0.00010077212937176228,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02289407141506672
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.203367743566718e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2814280986785889,
"saved_at_worst": 1.2814280986785889,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.364418029785156e-07,
"max_tolerance_ratio": 0.4608314751584185,
"worst_tolerance_index": [
758,
3
],
"actual_at_worst": -0.0016402006149291992,
"saved_at_worst": -0.0016407370567321777,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00044241477735340595,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046979136765003204
},
"reported": {
"count": 3136,
"observable_mse": 0.00044241492287255824,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046979136765003204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00013935392661416896
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.00042639728053472936,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04468761011958122
},
"reported": {
"count": 407,
"observable_mse": 0.0004263973096385598,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04468761011958122
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.7913601978153094e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0004414815630298108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0469791404902935
},
"reported": {
"count": 448,
"observable_mse": 0.00044148170854896307,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046979136765003204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006537990060863823
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.00044545609853230417,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0473763607442379
},
"reported": {
"count": 2281,
"observable_mse": 0.0004454563313629478,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0473763607442379
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00022290136138407402
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006795178167521954
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4145125280483626e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4145105271600187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9960688147343306e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006793184205889702
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4182987544918433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4183040295611136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.262343324217096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007076622452586889
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4742850655457005e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.474279244779609e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.806399437731063e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00707575399428606
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4824970751069486e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4825025320751593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.44345482026303e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02292351797223091
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.174026930239052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.173992005642503e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00034710295461388345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02197711542248726
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.503961438080296e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.5039843573467806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.2919266484677792e-10,
"numeric_tolerance_ratio": 0.0002279380971257544
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.022106481716036797
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.032768189674243e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.03276673448272e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4450289711579617e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.013866852968931198
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.329649189254269e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.3296459150733426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.26331524211703e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 6.998432908560917e-16
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006337612867355347
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7632882847683504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7633053832687438e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.709850039333105e-10,
"numeric_tolerance_ratio": 0.0001705138221705848
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.003950412385165691
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.004109410336241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0041121388203464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-11,
"numeric_tolerance_ratio": 2.7230268541460572e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218641808838584e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4531253780259127e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007395219057798386
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7916874387301505e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7916930775973015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.638867150992155e-11,
"numeric_tolerance_ratio": 5.623168989051261e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.465091529302299e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.266129628595387e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4603585188742727e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.460366340528708e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.821654435247183e-11,
"numeric_tolerance_ratio": 7.802457531363612e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.408349246252328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4083446987788193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-11,
"numeric_tolerance_ratio": 4.536547937688154e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4505914552719332e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4505910914740525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6290854027335757e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.732934990490321e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7329424483468756e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.457856554538012e-11,
"numeric_tolerance_ratio": 7.437530212509383e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.6724419512902386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.672434493433684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.457856554538012e-11,
"numeric_tolerance_ratio": 7.437979042782642e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.5223119589791168e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.5223105947370641e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3642420526593924e-11,
"numeric_tolerance_ratio": 1.3621684092581624e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5361989426310174e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5361960322479717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.90591898445941e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.4043375964974985e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.404349238029681e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011590483644883742
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 2.428454718028661e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 3.5170724004274234e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 3.5170724004274234e-06
},
"gradients": {
"norms": {
"prediction": 0.0001003925281111151,
"weighted_latent": 2.6831793128829187e-18,
"total": 0.0001003925281111151
},
"prediction_weighted_latent_dot": -2.871925118281443e-23,
"prediction_weighted_latent_cosine": -0.10661591151403256,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "bda2f8457e2047e74b2f4197dcf8e137d5b22a58fb906b015a5af31e57ab3bd4"
}
],
"final_losses": {
"prediction_mse": 2.428454718028661e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 3.5170724004274234e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 3.5170724004274234e-06
},
"training_seconds": 4.220003029331565e-05,
"evaluation_seconds": 0.6724775000475347
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 2,
"arm": "native",
"status": "passed",
"files": {
"seed-2-native/result.json": "1e4139794677c6b8a539f6fb9403c61baf33ba170deeaca82a72bffbf1ec06fc",
"seed-2-native/checkpoint.json": "580f3bfc531560d72eb00aabbae820bc1c8f7ce00904e4fd0199f6b3dcdc5cdd",
"inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"seed-2-native/diagnostics-arrays.npz": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.172325134277344e-07,
"max_tolerance_ratio": 0.38184041187270573,
"worst_tolerance_index": [
90,
15
],
"actual_at_worst": -0.0009272992610931396,
"saved_at_worst": -0.0009268820285797119,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.4284559913212433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.428454718028661e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2702079400172872e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926299571990967,
"saved_at_worst": 1.2926299571990967,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5302791797917392,
"worst_tolerance_index": [
545,
1
],
"actual_at_worst": -0.0012408345937728882,
"saved_at_worst": -0.0012402385473251343,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.00010886224481509998,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.048658352345228195
},
"reported": {
"count": 784,
"observable_mse": 0.00010886227391893044,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0486583486199379
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006350827096410438
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 0.00011503381392685696,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05235643684864044
},
"reported": {
"count": 580,
"observable_mse": 0.00011503385758260265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05235643684864044
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3159267976522494e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.00013099024363327771,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.048658352345228195
},
"reported": {
"count": 112,
"observable_mse": 0.00013099025818519294,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.048658352345228195
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4363763914006169e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 4.30161198892165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"reported": {
"count": 92,
"observable_mse": 4.301610169932246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.811198334370962e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4489330826327205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 98,
"observable_mse": 2.448930717946496e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749081823974848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333975094181
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0924091586493887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925644695759
},
"reported": {
"count": 70,
"observable_mse": 2.0924004275002517e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749084152281284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333754161296
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1905292669543996e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 14,
"observable_mse": 2.1905329049332067e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474907483905554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.792795956134796e-08,
"numeric_tolerance_ratio": 0.01330469955564529
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.489958519116044e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474924247711897
},
"reported": {
"count": 14,
"observable_mse": 4.489981074584648e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00347490725107491
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6996636986732483e-08,
"numeric_tolerance_ratio": 0.012613546549922739
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4650891646160744e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355574123561382
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017459385640157722
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0494757336564362e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 70,
"observable_mse": 2.049479189736303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355571795254946
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00034918772499636073
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.5227938749594614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335555549710989
},
"reported": {
"count": 14,
"observable_mse": 2.5227842343156226e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355604391545057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.0036664702162613125
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.485453246161342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 14,
"observable_mse": 4.485439421841875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355567138642073
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006983754743791358
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4724424292799085e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488371381536126
},
"reported": {
"count": 98,
"observable_mse": 2.4724389732000418e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883692860603333
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0954757928848267e-09,
"numeric_tolerance_ratio": 0.0015535427214692392
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9836363208014518e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488370683044195
},
"reported": {
"count": 70,
"observable_mse": 1.983637412195094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012083110264445502
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.7921978471567854e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883730113506317
},
"reported": {
"count": 14,
"observable_mse": 2.792183840938378e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488372080028057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006904632887422107
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.596718281391077e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883723128587008
},
"reported": {
"count": 14,
"observable_mse": 4.596703365677968e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0024166220528891004
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4542685423512012e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660863498225808
},
"reported": {
"count": 98,
"observable_mse": 2.4542630853829905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660864196717739
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0005113087436513964
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9375931515241973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036534646060317755
},
"reported": {
"count": 72,
"observable_mse": 1.9375880583538674e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00365346553735435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006821144214759868
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0936709663365036e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608625669032335
},
"reported": {
"count": 14,
"observable_mse": 3.093665145570412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608658265322447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002386107185697774
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8083511501317844e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704948117956519
},
"reported": {
"count": 12,
"observable_mse": 4.808346784557216e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704944159835577
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.958120942115784e-09,
"numeric_tolerance_ratio": 0.002888097095437761
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.454017158015631e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00366382603533566
},
"reported": {
"count": 98,
"observable_mse": 2.4540126105421223e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638197489082813
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004600783305236997
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9373417671886273e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656336572021246
},
"reported": {
"count": 72,
"observable_mse": 1.937336492119357e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656331915408373
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0034098562497762368
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.09398710669484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638290621340275
},
"reported": {
"count": 14,
"observable_mse": 3.0939725547796115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003663820680230856
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.0061343773221983556
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8074376536533237e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003708441974595189
},
"reported": {
"count": 12,
"observable_mse": 4.8074492951855063e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00370843056589365
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1408701539039612e-08,
"numeric_tolerance_ratio": 0.008322398019379602
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4505963665433228e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 98,
"observable_mse": 2.4505910914740525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0662389943026938e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 70,
"observable_mse": 2.066234446829185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.32611782848835e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416656516492367
},
"reported": {
"count": 14,
"observable_mse": 2.3261030946741812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416670486330986
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.010412299112111879
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.4968615839025006e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416659077629447
},
"reported": {
"count": 14,
"observable_mse": 4.4968630390940234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416667925193906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006594457363167671
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0007092258892953396,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"reported": {
"count": 98,
"observable_mse": 0.0007092263549566269,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00043482252767233174
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.000681952282320708,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"reported": {
"count": 84,
"observable_mse": 0.0006819527479819953,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0004359327346731717
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0008728672983124852,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"reported": {
"count": 14,
"observable_mse": 0.0008728685788810253,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1373080313205719
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.280568540096283e-09,
"numeric_tolerance_ratio": 0.001177765123165015
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218651813280303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703649388626218
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703637048602104
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.009004926261465663
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.265882656298345e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003696256782859564
},
"reported": {
"count": 72,
"observable_mse": 1.2658831110456958e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036962456069886684
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.008159806136714668
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4861635463603307e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703654045239091
},
"reported": {
"count": 14,
"observable_mse": 1.4861591807857621e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037036393769085407
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4668330550193787e-08,
"numeric_tolerance_ratio": 0.010703967133658529
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.2827452994533814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037476930301636457
},
"reported": {
"count": 12,
"observable_mse": 2.2827287466498092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00374767673201859
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-08,
"numeric_tolerance_ratio": 0.01185519951732077
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29456627368927,
"saved_at_worst": 1.29456627368927,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.48837328246516154,
"worst_tolerance_index": [
1481,
14
],
"actual_at_worst": 6.841123104095459e-05,
"saved_at_worst": 6.890296936035156e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.0004635656950995326,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07253267616033554
},
"reported": {
"count": 6272,
"observable_mse": 0.00046356572420336306,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07253267616033554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.7814447984412602e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.0005398132489062846,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07738631963729858
},
"reported": {
"count": 4092,
"observable_mse": 0.0005398132489062846,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07738631963729858
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.00046595203457400203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07253267616033554
},
"reported": {
"count": 896,
"observable_mse": 0.0004659521218854934,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.07253267616033554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.342431759039187e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.00021890609059482813,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.054240379482507706
},
"reported": {
"count": 1284,
"observable_mse": 0.00021890613425057381,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.054240379482507706
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.272056628329344e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2131738662719727,
"saved_at_worst": 1.2131738662719727,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.39050032273574337,
"worst_tolerance_index": [
1143,
0
],
"actual_at_worst": -0.00020797178149223328,
"saved_at_worst": -0.00020757317543029785,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0008681408362463117,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10035797953605652
},
"reported": {
"count": 3136,
"observable_mse": 0.0008681408944539726,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10035797208547592
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0006751284439290988
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.000930042820982635,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10425479710102081
},
"reported": {
"count": 1582,
"observable_mse": 0.0009300428791902959,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10425479710102081
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.3254741593273114e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0008559529087506235,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10035797208547592
},
"reported": {
"count": 448,
"observable_mse": 0.0008559530833736062,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.10035797208547592
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016085458494459172
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.0007845343207009137,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09450514614582062
},
"reported": {
"count": 1106,
"observable_mse": 0.0007845344371162355,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09450514614582062
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010794654373421804
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2567524909973145,
"saved_at_worst": 1.2567524909973145,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.3373281591691426,
"worst_tolerance_index": [
1728,
1
],
"actual_at_worst": 0.00015969574451446533,
"saved_at_worst": 0.00016003847122192383,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0070852432399988174,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.20626290142536163
},
"reported": {
"count": 3136,
"observable_mse": 0.00708524277433753,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.20626290142536163
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002725517532634505
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.008809944614768028,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.22894664108753204
},
"reported": {
"count": 1001,
"observable_mse": 0.008809943683445454,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.22894664108753204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004951224683544009
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0070871817879378796,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.20626290142536163
},
"reported": {
"count": 448,
"observable_mse": 0.007087182253599167,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.20626290142536163
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002725208172983901
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.006061358377337456,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1915375143289566
},
"reported": {
"count": 1687,
"observable_mse": 0.00606135930866003,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1915374994277954
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0007393741232353771
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2043378353118896,
"saved_at_worst": 1.2043378353118896,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.736947059631348e-07,
"max_tolerance_ratio": 0.3696263958661877,
"worst_tolerance_index": [
1766,
3
],
"actual_at_worst": -0.0006836354732513428,
"saved_at_worst": -0.0006832405924797058,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0199440885335207,
"exact_state_accuracy": 0.9205994897959183,
"per_field_accuracy": {
"x": 0.9652423469387755,
"y": 0.9719387755102041,
"h": 0.9792729591836735
},
"normalized_latent_rmse": 0.29997682571411133
},
"reported": {
"count": 3136,
"observable_mse": 0.0199440885335207,
"exact_state_accuracy": 0.9205994606018066,
"per_field_accuracy": {
"x": 0.965242326259613,
"y": 0.9719387292861938,
"h": 0.9792729616165161
},
"normalized_latent_rmse": 0.29997679591178894
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.622401028608891e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0009614372037117338
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.025644240900874138,
"exact_state_accuracy": 0.8771498771498771,
"per_field_accuracy": {
"x": 0.9656019656019657,
"y": 0.9533169533169533,
"h": 0.9533169533169533
},
"normalized_latent_rmse": 0.3439321219921112
},
"reported": {
"count": 407,
"observable_mse": 0.025644240900874138,
"exact_state_accuracy": 0.8771498799324036,
"per_field_accuracy": {
"x": 0.965601921081543,
"y": 0.9533169269561768,
"h": 0.9533169269561768
},
"normalized_latent_rmse": 0.3439320921897888
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.452042268177081e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0008420350413353993
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.020050769671797752,
"exact_state_accuracy": 0.9196428571428571,
"per_field_accuracy": {
"x": 0.9665178571428571,
"y": 0.9709821428571429,
"h": 0.9799107142857143
},
"normalized_latent_rmse": 0.29997679591178894
},
"reported": {
"count": 448,
"observable_mse": 0.0200507715344429,
"exact_state_accuracy": 0.9196429252624512,
"per_field_accuracy": {
"x": 0.9665179252624512,
"y": 0.9709821939468384,
"h": 0.9799107313156128
},
"normalized_latent_rmse": 0.29997679591178894
},
"differences": [],
"max_errors": {
"accuracy_abs": 6.811959407659884e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006198327211319927
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.01890605501830578,
"exact_state_accuracy": 0.9285401139850943,
"per_field_accuracy": {
"x": 0.9649276633055678,
"y": 0.9754493643138974,
"h": 0.9837790442788251
},
"normalized_latent_rmse": 0.29143768548965454
},
"reported": {
"count": 2281,
"observable_mse": 0.01890605501830578,
"exact_state_accuracy": 0.9285401105880737,
"per_field_accuracy": {
"x": 0.9649276733398438,
"y": 0.9754493832588196,
"h": 0.9837790727615356
},
"normalized_latent_rmse": 0.29143771529197693
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.8482710523469734e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0009886726469787747
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006795178167521954
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4145125280483626e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4145105271600187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9960688147343306e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006793184205889702
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4182987544918433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4183040295611136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.262343324217096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007076622452586889
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4742850655457005e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.474279244779609e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.806399437731063e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00707575399428606
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4824970751069486e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4825025320751593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.44345482026303e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02292351797223091
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.174026930239052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.173992005642503e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00034710295461388345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02197711542248726
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.503961438080296e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.5039843573467806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.2919266484677792e-10,
"numeric_tolerance_ratio": 0.0002279380971257544
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.022106481716036797
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.032768189674243e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.03276673448272e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4450289711579617e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.013866852968931198
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.329649189254269e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.3296459150733426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.26331524211703e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.23434017598628998
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.011553759686648846,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.01155376061797142,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.000432092845013295
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.2341216802597046
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.013623347505927086,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.013623349368572235,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0007884763164485735
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.003950412385165691
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.004109410336241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0041121388203464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-11,
"numeric_tolerance_ratio": 2.7230268541460572e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218641808838584e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4531253780259127e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007395219057798386
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7916874387301505e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7916930775973015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.638867150992155e-11,
"numeric_tolerance_ratio": 5.623168989051261e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.465091529302299e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.266129628595387e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4603585188742727e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.460366340528708e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.821654435247183e-11,
"numeric_tolerance_ratio": 7.802457531363612e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.408349246252328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4083446987788193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-11,
"numeric_tolerance_ratio": 4.536547937688154e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0020825800020247698,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0020825802348554134,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00019269943929874165
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0007546067936345935,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0007546074921265244,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0006494815654341421
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.6724419512902386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.672434493433684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.457856554538012e-11,
"numeric_tolerance_ratio": 7.437979042782642e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.5223119589791168e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.5223105947370641e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3642420526593924e-11,
"numeric_tolerance_ratio": 1.3621684092581624e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5361989426310174e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5361960322479717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.90591898445941e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.4043375964974985e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.404349238029681e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011590483644883742
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 636,
"error_count": 50,
"per_field_correct_counts": {
"x": 682,
"y": 640,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 441,
"error_count": 49,
"per_field_correct_counts": {
"x": 441,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 91,
"error_count": 7,
"per_field_correct_counts": {
"x": 98,
"y": 91,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 140,
"error_count": 56,
"per_field_correct_counts": {
"x": 140,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 74,
"error_count": 24,
"per_field_correct_counts": {
"x": 98,
"y": 84,
"h": 84
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 35,
"error_count": 63,
"per_field_correct_counts": {
"x": 98,
"y": 77,
"h": 47
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 2887,
"accuracy": 0.9205994897959183
},
"numpy": {
"cases": 3136,
"correct": 2887,
"accuracy": 0.9205994897959183
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0006819527479819953,
"latent_mse": 0.01885349676012993,
"scaled_prediction": 9.876557305688038e-05,
"scaled_weighted_latent": 0.0002730506530497223,
"total": 0.0003718162188306451
},
"gradients": {
"norms": {
"prediction": 0.0009655424510128796,
"weighted_latent": 0.001503756851889193,
"total": 0.002251012483611703
},
"prediction_weighted_latent_dot": 9.367491315970256e-07,
"prediction_weighted_latent_cosine": 0.6451702116278689,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "580f3bfc531560d72eb00aabbae820bc1c8f7ce00904e4fd0199f6b3dcdc5cdd"
}
],
"final_losses": {
"prediction_mse": 0.0006819527479819953,
"latent_mse": 0.01885349676012993,
"scaled_prediction": 9.876557305688038e-05,
"scaled_weighted_latent": 0.0002730506530497223,
"total": 0.0003718162188306451
},
"training_seconds": 4.299997817724943e-05,
"evaluation_seconds": 0.5754795999964699
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 2,
"arm": "prediction_latent",
"status": "passed",
"files": {
"seed-2-prediction_latent/result.json": "3c6dd3dc32279410c2d961c6fca25eccac4ec596e4d54b8e89de094c21fd5072",
"seed-2-prediction_latent/checkpoint.json": "96fb11342d85934fa4a7978a0ab5c3a67fc936118bd303aa84569e9dc2413884",
"inputs/seed-2-fourier/checkpoint.json": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"inputs/seed-2-fourier/result.json": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"seed-2-prediction_latent/diagnostics-arrays.npz": "76c4e3d623aa6a3635ab5ebfaa25058b43e9e885408a610ea856200f7b23f635"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d01b798a8d74e3b373db2f04254710a0fcf73d07f014eedb61bad3ca7174aef5",
"parent_checkpoint_sha256": "86d95c3c3c98fe4e0e37be0f6794ddf50e51cc7341259689fd34faa29feaac34",
"parent_arrays_sha256": "d7eb0db9adf46bb798c642cad1ebd06df7b9cbe88a21b94326de9bc574002105",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.172325134277344e-07,
"max_tolerance_ratio": 0.38184041187270573,
"worst_tolerance_index": [
90,
15
],
"actual_at_worst": -0.0009272992610931396,
"saved_at_worst": -0.0009268820285797119,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.4284559913212433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.428454718028661e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2702079400172872e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 4.753383473143913e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2926299571990967,
"saved_at_worst": 1.2926299571990967,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5302791797917392,
"worst_tolerance_index": [
545,
1
],
"actual_at_worst": -0.0012408345937728882,
"saved_at_worst": -0.0012402385473251343,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.326729372725822e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034997230395674706
},
"reported": {
"count": 784,
"observable_mse": 2.3267271899385378e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034997230395674706
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1777203231670335e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.9321074432809837e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00349230645224452
},
"reported": {
"count": 580,
"observable_mse": 1.9321063518873416e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034923062194138765
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017256549018940373
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.748085171333514e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034997237380594015
},
"reported": {
"count": 112,
"observable_mse": 2.7480740754981525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034997237380594015
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1095835361629725e-10,
"numeric_tolerance_ratio": 0.000110654267492458
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 4.30161198892165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"reported": {
"count": 92,
"observable_mse": 4.301610169932246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035461238585412502
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.811198334370962e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4489330826327205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 98,
"observable_mse": 2.448930717946496e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749081823974848
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333975094181
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0924091586493887e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925644695759
},
"reported": {
"count": 70,
"observable_mse": 2.0924004275002517e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034749084152281284
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7229467630386353e-08,
"numeric_tolerance_ratio": 0.012786333754161296
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1905292669543996e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474925411865115
},
"reported": {
"count": 14,
"observable_mse": 2.1905329049332067e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474907483905554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.792795956134796e-08,
"numeric_tolerance_ratio": 0.01330469955564529
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.489958519116044e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003474924247711897
},
"reported": {
"count": 14,
"observable_mse": 4.489981074584648e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00347490725107491
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6996636986732483e-08,
"numeric_tolerance_ratio": 0.012613546549922739
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4650891646160744e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355574123561382
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017459385640157722
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0494757336564362e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 70,
"observable_mse": 2.049479189736303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355571795254946
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00034918772499636073
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.5227938749594614e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335555549710989
},
"reported": {
"count": 14,
"observable_mse": 2.5227842343156226e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355604391545057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.0036664702162613125
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.485453246161342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003335557645186782
},
"reported": {
"count": 14,
"observable_mse": 4.485439421841875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033355567138642073
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006983754743791358
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4724424292799085e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488371381536126
},
"reported": {
"count": 98,
"observable_mse": 2.4724389732000418e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883692860603333
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0954757928848267e-09,
"numeric_tolerance_ratio": 0.0015535427214692392
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9836363208014518e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488370683044195
},
"reported": {
"count": 70,
"observable_mse": 1.983637412195094e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012083110264445502
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.7921978471567854e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883730113506317
},
"reported": {
"count": 14,
"observable_mse": 2.792183840938378e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003488372080028057
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006904632887422107
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.596718281391077e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883723128587008
},
"reported": {
"count": 14,
"observable_mse": 4.596703365677968e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034883690532296896
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0024166220528891004
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4542685423512012e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660863498225808
},
"reported": {
"count": 98,
"observable_mse": 2.4542630853829905e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003660864196717739
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-10,
"numeric_tolerance_ratio": 0.0005113087436513964
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9375931515241973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036534646060317755
},
"reported": {
"count": 72,
"observable_mse": 1.9375880583538674e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00365346553735435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0006821144214759868
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.0936709663365036e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608625669032335
},
"reported": {
"count": 14,
"observable_mse": 3.093665145570412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036608658265322447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.002386107185697774
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8083511501317844e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704948117956519
},
"reported": {
"count": 12,
"observable_mse": 4.808346784557216e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003704944159835577
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.958120942115784e-09,
"numeric_tolerance_ratio": 0.002888097095437761
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.454017158015631e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00366382603533566
},
"reported": {
"count": 98,
"observable_mse": 2.4540126105421223e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638197489082813
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004600783305236997
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.9373417671886273e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656336572021246
},
"reported": {
"count": 72,
"observable_mse": 1.937336492119357e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003656331915408373
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.6566128730773926e-09,
"numeric_tolerance_ratio": 0.0034098562497762368
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 3.09398710669484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036638290621340275
},
"reported": {
"count": 14,
"observable_mse": 3.0939725547796115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003663820680230856
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.0061343773221983556
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.8074376536533237e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003708441974595189
},
"reported": {
"count": 12,
"observable_mse": 4.8074492951855063e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00370843056589365
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1408701539039612e-08,
"numeric_tolerance_ratio": 0.008322398019379602
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4505963665433228e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 98,
"observable_mse": 2.4505910914740525e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0662389943026938e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034166586119681597
},
"reported": {
"count": 70,
"observable_mse": 2.066234446829185e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416669089347124
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.00780922514720379
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.32611782848835e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416656516492367
},
"reported": {
"count": 14,
"observable_mse": 2.3261030946741812e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416670486330986
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.010412299112111879
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.4968615839025006e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416659077629447
},
"reported": {
"count": 14,
"observable_mse": 4.4968630390940234e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003416667925193906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006594457363167671
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4466247850796208e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032246853224933147
},
"reported": {
"count": 98,
"observable_mse": 2.4466211471008137e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032246781047433615
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.2177499532699585e-09,
"numeric_tolerance_ratio": 0.005457788761362088
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.1078587451484054e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032246853224933147
},
"reported": {
"count": 84,
"observable_mse": 2.1078567442600615e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032246774062514305
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.916241884231567e-09,
"numeric_tolerance_ratio": 0.005985962183462777
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 4.479219933273271e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032246848568320274
},
"reported": {
"count": 14,
"observable_mse": 4.47920938313473e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003224681131541729
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0028169225869475625
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218651813280303e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703649388626218
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703637048602104
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.009004926261465663
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.265882656298345e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003696256782859564
},
"reported": {
"count": 72,
"observable_mse": 1.2658831110456958e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0036962456069886684
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.008159806136714668
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4861635463603307e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003703654045239091
},
"reported": {
"count": 14,
"observable_mse": 1.4861591807857621e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037036393769085407
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4668330550193787e-08,
"numeric_tolerance_ratio": 0.010703967133658529
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 2.2827452994533814e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0037476930301636457
},
"reported": {
"count": 12,
"observable_mse": 2.2827287466498092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00374767673201859
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-08,
"numeric_tolerance_ratio": 0.01185519951732077
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29456627368927,
"saved_at_worst": 1.29456627368927,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5081765317664586,
"worst_tolerance_index": [
4802,
10
],
"actual_at_worst": -0.00041012465953826904,
"saved_at_worst": -0.00040959566831588745,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.381150443397928e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006720003671944141
},
"reported": {
"count": 6272,
"observable_mse": 2.381150443397928e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006720003206282854
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00027850550120274995
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 2.00920276256511e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006727665662765503
},
"reported": {
"count": 4092,
"observable_mse": 2.0092016711714678e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006727665662765503
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0892052092009387e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.616950405354146e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006720003671944141
},
"reported": {
"count": 896,
"observable_mse": 2.6169513148488477e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006720003671944141
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.071208107744453e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.401971480343491e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006695527117699385
},
"reported": {
"count": 1284,
"observable_mse": 3.401972571737133e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.006695527117699385
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0876933392210228e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2936732769012451,
"saved_at_worst": 1.2936732769012451,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4389024947856207,
"worst_tolerance_index": [
893,
11
],
"actual_at_worst": 0.00026971474289894104,
"saved_at_worst": 0.00027016550302505493,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.829687193501741e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013225768692791462
},
"reported": {
"count": 3136,
"observable_mse": 3.8296886486932635e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013225768692791462
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4496398535448712e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 3.53561008523684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013272375799715519
},
"reported": {
"count": 1582,
"observable_mse": 3.535611540428363e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013272375799715519
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4500646574992635e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.879581345245242e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013225769624114037
},
"reported": {
"count": 448,
"observable_mse": 3.879580617649481e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.013225769624114037
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.247839038330476e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 4.2301195207983255e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01315881498157978
},
"reported": {
"count": 1106,
"observable_mse": 4.2301209759898484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01315881498157978
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4490618160530932e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2936776876449585,
"saved_at_worst": 1.2936776876449585,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.556510925292969e-07,
"max_tolerance_ratio": 0.4808236477509496,
"worst_tolerance_index": [
1653,
3
],
"actual_at_worst": -0.00038145482540130615,
"saved_at_worst": -0.00038195401430130005,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0001303325843764469,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026330674067139626
},
"reported": {
"count": 3136,
"observable_mse": 0.00013033264258410782,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026330672204494476
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.000512692178869327
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00013152690371498466,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02676781266927719
},
"reported": {
"count": 1001,
"observable_mse": 0.00013152696192264557,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02676781266927719
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.745201205329609e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0001308084320044145,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026330670341849327
},
"reported": {
"count": 448,
"observable_mse": 0.00013080847566016018,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.026330670341849327
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3092064952156534e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00012949758092872798,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02606782503426075
},
"reported": {
"count": 1687,
"observable_mse": 0.00012949762458447367,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02606782503426075
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.309764146560169e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2798446416854858,
"saved_at_worst": 1.2798446416854858,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4056835508193834,
"worst_tolerance_index": [
1762,
3
],
"actual_at_worst": -0.0010197460651397705,
"saved_at_worst": -0.001019299030303955,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0005900115356780589,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05305289849638939
},
"reported": {
"count": 3136,
"observable_mse": 0.0005900117103010416,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05305289849638939
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016489404121295182
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0006105774082243443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.052701715379953384
},
"reported": {
"count": 407,
"observable_mse": 0.000610577582847327,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.052701715379953384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016457443657232326
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0005885306745767593,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05305289477109909
},
"reported": {
"count": 448,
"observable_mse": 0.0005885308491997421,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05305289477109909
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.0001649171025020906
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0005866328137926757,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05311531946063042
},
"reported": {
"count": 2281,
"observable_mse": 0.0005866330466233194,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.05311531573534012
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0005902355482277996
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006795178167521954
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4145125280483626e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4145105271600187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9960688147343306e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006793184205889702
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4182987544918433e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4183040295611136e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.262343324217096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007076622452586889
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4742850655457005e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.474279244779609e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.806399437731063e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00707575399428606
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4824970751069486e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4825025320751593e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-11,
"numeric_tolerance_ratio": 5.44345482026303e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02292351797223091
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.174026930239052e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.173992005642503e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00034710295461388345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.02197711542248726
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 5.503961438080296e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 5.5039843573467806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.2919266484677792e-10,
"numeric_tolerance_ratio": 0.0002279380971257544
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.022106481716036797
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.032768189674243e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.03276673448272e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4450289711579617e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.013866852968931198
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.329649189254269e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.3296459150733426e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.26331524211703e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006114483345299959
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7247950129094534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7247851903666742e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.822542779147625e-11,
"numeric_tolerance_ratio": 9.795851188901073e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012204460799694061
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.575491766445339e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.575491766445339e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7605883587966673e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7605872674030252e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0883890491763767e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.003950412385165691
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.004109410336241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.0041121388203464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7284841053187847e-11,
"numeric_tolerance_ratio": 2.7230268541460572e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4218641808838584e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4218627256923355e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4531253780259127e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.007395219057798386
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7916874387301505e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7916930775973015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.638867150992155e-11,
"numeric_tolerance_ratio": 5.623168989051261e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.465091529302299e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4650882551213726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.2741809263825417e-11,
"numeric_tolerance_ratio": 3.266129628595387e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.4603585188742727e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.460366340528708e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.821654435247183e-11,
"numeric_tolerance_ratio": 7.802457531363612e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.408349246252328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.4083446987788193e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.547473508864641e-11,
"numeric_tolerance_ratio": 4.536547937688154e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.468158709234558e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.468152524670586e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.184563972055912e-11,
"numeric_tolerance_ratio": 6.169337107099482e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7641772248898633e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7641910492093302e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.00013786211743843673
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.6724419512902386e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.672434493433684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.457856554538012e-11,
"numeric_tolerance_ratio": 7.437979042782642e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.5223119589791168e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.5223105947370641e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3642420526593924e-11,
"numeric_tolerance_ratio": 1.3621684092581624e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5361989426310174e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.5361960322479717e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.90591898445941e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.4043375964974985e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.404349238029681e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011590483644883742
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0006819527479819953,
"latent_mse": 0.01885349676012993,
"scaled_prediction": 9.876557305688038e-05,
"scaled_weighted_latent": 0.0002730506530497223,
"total": 0.0003718162188306451
},
"gradients": {
"norms": {
"prediction": 0.0009655424510128796,
"weighted_latent": 0.001503756851889193,
"total": 0.002251012483611703
},
"prediction_weighted_latent_dot": 9.367491315970256e-07,
"prediction_weighted_latent_cosine": 0.6451702116278689,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0007664921577088535,
"latent_mse": 0.017893053591251373,
"scaled_prediction": 0.00011100921256002039,
"scaled_weighted_latent": 0.000259140768321231,
"total": 0.00037014996632933617
},
"gradients": {
"norms": {
"prediction": 0.0016901097260415554,
"weighted_latent": 0.0014657382853329182,
"total": 0.0026715160347521305
},
"prediction_weighted_latent_dot": 1.0660676252882695e-06,
"prediction_weighted_latent_cosine": 0.4303416908538793,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.02936518006026745
},
{
"step": 100,
"losses": {
"prediction_mse": 2.6542955311015248e-05,
"latent_mse": 0.000432105443906039,
"scaled_prediction": 3.844152161036618e-06,
"scaled_weighted_latent": 6.258078883547569e-06,
"total": 1.0102230589836836e-05
},
"gradients": {
"norms": {
"prediction": 0.00014162296429276466,
"weighted_latent": 0.0002406078710919246,
"total": 0.0003380496636964381
},
"prediction_weighted_latent_dot": 1.8164215020988195e-08,
"prediction_weighted_latent_cosine": 0.5330563527859227,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.001268652849830687
},
{
"step": 200,
"losses": {
"prediction_mse": 2.0889330698992126e-05,
"latent_mse": 1.2319784218561836e-05,
"scaled_prediction": 3.0253513614297844e-06,
"scaled_weighted_latent": 1.7842447164184705e-07,
"total": 3.203775804649922e-06
},
"gradients": {
"norms": {
"prediction": 3.2663530873833224e-05,
"weighted_latent": 3.844348975690082e-05,
"total": 6.885149105073651e-06
},
"prediction_weighted_latent_dot": -1.2487013645312572e-09,
"prediction_weighted_latent_cosine": -0.9944264160208769,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.3797200962435454e-05
},
{
"step": 300,
"losses": {
"prediction_mse": 2.107463478751015e-05,
"latent_mse": 1.0362839930166956e-05,
"scaled_prediction": 3.0521885037160246e-06,
"scaled_weighted_latent": 1.5008251352810476e-07,
"total": 3.2022710456658388e-06
},
"gradients": {
"norms": {
"prediction": 3.676263804663904e-05,
"weighted_latent": 3.492542600724846e-05,
"total": 1.2157050150563009e-05
},
"prediction_weighted_latent_dot": -1.211741595952276e-09,
"prediction_weighted_latent_cosine": -0.9437601508857996,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00044254420208744705
},
{
"step": 400,
"losses": {
"prediction_mse": 2.1073601601528935e-05,
"latent_mse": 1.0396162906545214e-05,
"scaled_prediction": 3.052038891837583e-06,
"scaled_weighted_latent": 1.5056512836508773e-07,
"total": 3.2026039207266876e-06
},
"gradients": {
"norms": {
"prediction": 3.8248101191129535e-05,
"weighted_latent": 3.5035762266488746e-05,
"total": 1.6219282770180143e-05
},
"prediction_weighted_latent_dot": -1.2136783800187345e-09,
"prediction_weighted_latent_cosine": -0.9056954066841081,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005452635232359171
},
{
"step": 500,
"losses": {
"prediction_mse": 2.1075411495985463e-05,
"latent_mse": 1.0414029020466842e-05,
"scaled_prediction": 3.052301053685369e-06,
"scaled_weighted_latent": 1.508238653968874e-07,
"total": 3.203124833817128e-06
},
"gradients": {
"norms": {
"prediction": 3.8674745155731216e-05,
"weighted_latent": 3.5055403714068234e-05,
"total": 1.786229040590115e-05
},
"prediction_weighted_latent_dot": -1.2027779883183598e-09,
"prediction_weighted_latent_cosine": -0.8871622178713677,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005519929109141231
},
{
"step": 600,
"losses": {
"prediction_mse": 2.1348816517274827e-05,
"latent_mse": 1.0642489542078692e-05,
"scaled_prediction": 3.091897724516457e-06,
"scaled_weighted_latent": 1.5413260712193733e-07,
"total": 3.246030246373266e-06
},
"gradients": {
"norms": {
"prediction": 8.918772800825536e-05,
"weighted_latent": 3.561050471034832e-05,
"total": 8.645343041280285e-05
},
"prediction_weighted_latent_dot": -8.741819490509783e-10,
"prediction_weighted_latent_cosine": -0.27524447161999827,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0025028504896909
},
{
"step": 700,
"losses": {
"prediction_mse": 2.1100107915117405e-05,
"latent_mse": 1.03705060610082e-05,
"scaled_prediction": 3.0558778689737665e-06,
"scaled_weighted_latent": 1.5019354293599463e-07,
"total": 3.206071369277197e-06
},
"gradients": {
"norms": {
"prediction": 4.425676161190495e-05,
"weighted_latent": 3.4928660170407966e-05,
"total": 2.9300932510523126e-05
},
"prediction_weighted_latent_dot": -1.1600639338027463e-09,
"prediction_weighted_latent_cosine": -0.75044758746988,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0008021992980502546
},
{
"step": 800,
"losses": {
"prediction_mse": 2.106979809468612e-05,
"latent_mse": 1.0369310075475369e-05,
"scaled_prediction": 3.051487965421984e-06,
"scaled_weighted_latent": 1.501762199040968e-07,
"total": 3.2016641853260808e-06
},
"gradients": {
"norms": {
"prediction": 3.6618766898754984e-05,
"weighted_latent": 3.494641714496538e-05,
"total": 1.1658473340503406e-05
},
"prediction_weighted_latent_dot": -1.213133149491341e-09,
"prediction_weighted_latent_cosine": -0.9479863801021514,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0002748027036432177
},
{
"step": 900,
"losses": {
"prediction_mse": 2.1071549781481735e-05,
"latent_mse": 1.0367329196014907e-05,
"scaled_prediction": 3.0517417144437786e-06,
"scaled_weighted_latent": 1.501475281884268e-07,
"total": 3.2018892852647696e-06
},
"gradients": {
"norms": {
"prediction": 3.6724366509588435e-05,
"weighted_latent": 3.4931050322484225e-05,
"total": 1.1596196600294206e-05
},
"prediction_weighted_latent_dot": -1.217192679980883e-09,
"prediction_weighted_latent_cosine": -0.9488408513342776,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00042341844527982175
},
{
"step": 1000,
"losses": {
"prediction_mse": 2.1078567442600615e-05,
"latent_mse": 1.0398544873169158e-05,
"scaled_prediction": 3.05275807477301e-06,
"scaled_weighted_latent": 1.5059961810948153e-07,
"total": 3.203357664460782e-06
},
"gradients": {
"norms": {
"prediction": 4.1895014874171466e-05,
"weighted_latent": 3.5039363865507767e-05,
"total": 2.398364267719444e-05
},
"prediction_weighted_latent_dot": -1.2038667840386097e-09,
"prediction_weighted_latent_cosine": -0.8200868913974374,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00048400292871519923
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "363d6312324e6bb76c3505974198defafeb076e7a6dfcca3f6e995b0d6f68314"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "1a29db21b70eb2808fb52550476aa65f77fa3d52d5de26d2249720f212fd80a5"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "6e15fc9c2fc9d48920f0431193a8cc1071fcf7115c1ca38d71c03dd8b2ae300a"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "ad49ebd86692985f6c1dadc11727748c9564df2512f101feeb2fc3eea7d62bf6"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "86c8a5a1f8fd4080cd433c2018d0993cbebc742afe86cdfa587daa67f2437e14"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "fb0f041f35ad073664da9a86fae79a5c8b697268d0036fde78cd5d8a3d85102b"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "841fb220944a4bf864e696696022cc9c9eba9455fcfb870b8e712db105fc6e0c"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "4d1a6f7e75cc98493e3ecece3c766a78797c457bb6a1b16f8b0693871fed9949"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "f05cda5d84c6b8475222308b8c1ae14fc30e1b919a04f68afad3e408caefbd77"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "2df653451c9b3123ba08535fba3920cab58b1bf22e650e5b747eb3eae7150e64"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "5c34ee19a38584fb4f53325f797a694edd412edb11a1396bebd0abcaa30380d1"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "96fb11342d85934fa4a7978a0ab5c3a67fc936118bd303aa84569e9dc2413884"
}
],
"final_losses": {
"prediction_mse": 2.1078567442600615e-05,
"latent_mse": 1.0398544873169158e-05,
"scaled_prediction": 3.05275807477301e-06,
"scaled_weighted_latent": 1.5059961810948153e-07,
"total": 3.203357664460782e-06
},
"training_seconds": 3.3037349000223912,
"evaluation_seconds": 0.507699900015723
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 3,
"arm": "refit",
"status": "passed",
"files": {
"seed-3-refit/result.json": "9532d71b6cfd1249296a2382024f3026133c0435bd7b943d96faf9d9d97cd910",
"seed-3-refit/checkpoint.json": "e10d1479ab2365c43ffe7e27c9b1efa8384babeabb8f6328e4de58db8d11fd16",
"inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"seed-3-refit/diagnostics-arrays.npz": "cb4c22da61b396601351d7f1dae4ffed51547a2fcfacb19a754fd123b21b0682"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 5.811452865600586e-07,
"max_tolerance_ratio": 0.3990840958602454,
"worst_tolerance_index": [
60,
2
],
"actual_at_worst": 0.004562556743621826,
"saved_at_worst": 0.004561975598335266,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.00010696637036744505,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 0.00010696651588659734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014397905845926648
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001894764427561313,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 0.00018947666103485972,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-10,
"numeric_tolerance_ratio": 0.00021421976386698359
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2893109321594238,
"saved_at_worst": 1.2893109321594238,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.3990840958602454,
"worst_tolerance_index": [
649,
2
],
"actual_at_worst": 0.004562556743621826,
"saved_at_worst": 0.004561975598335266,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.00010002128692576662,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"reported": {
"count": 784,
"observable_mse": 0.00010002134513342753,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.763122564242308e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 8.43349116621539e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005999956279993057
},
"reported": {
"count": 580,
"observable_mse": 8.433497714577243e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0059999567456543446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002910390913614281
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.00011633474059635773,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"reported": {
"count": 112,
"observable_mse": 0.00011633482063189149,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.911514908817387e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 0.00017905373533722013,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"reported": {
"count": 92,
"observable_mse": 0.00017905370623338968,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8591882208963364e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.000102343074104283,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"reported": {
"count": 98,
"observable_mse": 0.00010234319051960483,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001757528884955003
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.67251874296926e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897133611142635
},
"reported": {
"count": 70,
"observable_mse": 8.67253402248025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0020504503657808367
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010630184988258407,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897137802094221
},
"reported": {
"count": 14,
"observable_mse": 0.00010630159522406757,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002343372258461114
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017647375352680683,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897135008126497
},
"reported": {
"count": 14,
"observable_mse": 0.00017647400090936571,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897139199078083
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002636292941319088
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252822539769113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.737066673347726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 70,
"observable_mse": 8.737093594390899e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010268489859299734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 14,
"observable_mse": 0.0001026846730383113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.255546860396862e-10,
"numeric_tolerance_ratio": 0.00022326212619665206
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017815936007536948,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"reported": {
"count": 14,
"observable_mse": 0.00017815965111367404,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002859439373556026
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001030568455462344,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580707758665
},
"reported": {
"count": 98,
"observable_mse": 0.00010305678733857349,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00579952893372632
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.592774247517809e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580242097378
},
"reported": {
"count": 70,
"observable_mse": 8.592770609539002e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585904866456985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.006379481642109011
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001099406072171405,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585797764360905
},
"reported": {
"count": 14,
"observable_mse": 0.00010994031617883593,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585867613554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.004349647583201486
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00018181865743827075,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058583967387676
},
"reported": {
"count": 14,
"observable_mse": 0.00018181874474976212,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003769693806922108
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200838732998818,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097275298088789
},
"reported": {
"count": 98,
"observable_mse": 0.00010200848191743717,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097279489040375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002603515450311345
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543613512301818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060906685888767
},
"reported": {
"count": 72,
"observable_mse": 8.543617150280625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060910876840353
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026094108969952367
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001222111313836649,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00609727343544364
},
"reported": {
"count": 14,
"observable_mse": 0.00012221158249303699,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060972776263952255
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026035157515688335
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00017787207616493106,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311086472123861
},
"reported": {
"count": 12,
"observable_mse": 0.00017787201795727015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311097647994757
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006851697621195759
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200543329119682,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094702053815126
},
"reported": {
"count": 98,
"observable_mse": 0.00010200551332673058,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094708573073149
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004050559842515715
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543937292415649e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058296654373407
},
"reported": {
"count": 72,
"observable_mse": 8.543945295969024e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058302707970142
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.0037697612537818566
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00012222511577419937,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060947067104279995
},
"reported": {
"count": 14,
"observable_mse": 0.00012222547957208008,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094712298363447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0034719076328323475
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.0001778121804818511,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006308719515800476
},
"reported": {
"count": 12,
"observable_mse": 0.0001778119767550379,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0063087306916713715
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006852692037580273
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067623384296894
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0023185078919724916
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.603723108535632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067622918635607
},
"reported": {
"count": 70,
"observable_mse": 8.603732567280531e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.00202869440547593
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010919466149061918,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067620124667883
},
"reported": {
"count": 14,
"observable_mse": 0.00010919448686763644,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028981346969819097
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017808005213737488,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067628972232342
},
"reported": {
"count": 14,
"observable_mse": 0.00017807971744332463,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.005216642454567437
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.0001069663412636146,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"reported": {
"count": 84,
"observable_mse": 0.00010696651588659734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00017277487015111977
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001894764427561313,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360026644082518e-16
},
"reported": {
"count": 14,
"observable_mse": 0.00018947666103485972,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360023997104558e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-10,
"numeric_tolerance_ratio": 0.00021421976386698359
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.69803557684645e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705172687768936
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.005974748847249882
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 4.724697646452114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754040114581585
},
"reported": {
"count": 72,
"observable_mse": 4.724687460111454e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754054084420204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.007448969996752607
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.864315218990669e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 14,
"observable_mse": 6.864342140033841e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705178275704384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6763806343078613e-08,
"numeric_tolerance_ratio": 0.008962120593553839
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00018344067211728543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405925706028938
},
"reported": {
"count": 12,
"observable_mse": 0.00018344077398069203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405914530158043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.0060718911179741195
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869952917099,
"saved_at_worst": 1.2869952917099,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.6967987208633615,
"worst_tolerance_index": [
4903,
3
],
"actual_at_worst": -0.0002655982971191406,
"saved_at_worst": -0.00026488304138183594,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.00011185490438947454,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010564569383859634
},
"reported": {
"count": 6272,
"observable_mse": 0.000111854933493305,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010564569383859634
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8781890808513913e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 9.627988038118929e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010497652925550938
},
"reported": {
"count": 4092,
"observable_mse": 9.627990948501974e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010497653856873512
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004543556941289534
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.0001191547344205901,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010564569383859634
},
"reported": {
"count": 896,
"observable_mse": 0.00011915479262825102,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010564569383859634
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.752225566889378e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.0001563972036819905,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010775051079690456
},
"reported": {
"count": 1284,
"observable_mse": 0.0001563972473377362,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.010775051079690456
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.298349564511559e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2947795391082764,
"saved_at_worst": 1.2947795391082764,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4723996082751296,
"worst_tolerance_index": [
844,
2
],
"actual_at_worst": -0.0007253140211105347,
"saved_at_worst": -0.0007248073816299438,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00019038809114135802,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01795356720685959
},
"reported": {
"count": 3136,
"observable_mse": 0.00019038810569327325,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01795356720685959
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.428004024717845e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.00017765552911441773,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018069889396429062
},
"reported": {
"count": 1582,
"observable_mse": 0.00017765557277016342,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01806989125907421
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006635740523678787
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00019056950986851007,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017953569069504738
},
"reported": {
"count": 448,
"observable_mse": 0.0001905695244204253,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017953569069504738
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4279786025202036e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.00020852700981777161,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01778586208820343
},
"reported": {
"count": 1106,
"observable_mse": 0.00020852700981777161,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01778586022555828
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006703572011485321
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2878199815750122,
"saved_at_worst": 1.2878199815750122,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.41482060580750596,
"worst_tolerance_index": [
3035,
3
],
"actual_at_worst": 0.003649771213531494,
"saved_at_worst": 0.0036503374576568604,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00040838465793058276,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028023647144436836
},
"reported": {
"count": 3136,
"observable_mse": 0.0004083846288267523,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028023645281791687
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0004898649604547301
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.0003915227425750345,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02775302715599537
},
"reported": {
"count": 1001,
"observable_mse": 0.00039152277167886496,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02775302715599537
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8007281604630173e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0004061652871314436,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028023649007081985
},
"reported": {
"count": 448,
"observable_mse": 0.0004061653162352741,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028023649007081985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.7967872479718438e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.0004189791507087648,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028182994574308395
},
"reported": {
"count": 1687,
"observable_mse": 0.0004189791507087648,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028182994574308395
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.328358769416809,
"saved_at_worst": 1.328358769416809,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.109476089477539e-07,
"max_tolerance_ratio": 0.379329818060494,
"worst_tolerance_index": [
1250,
2
],
"actual_at_worst": -0.0006059706211090088,
"saved_at_worst": -0.0006063729524612427,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0009452107478864491,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04636046662926674
},
"reported": {
"count": 3136,
"observable_mse": 0.0009452107478864491,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04636046290397644
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006609758164706417
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0008695956785231829,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04342574626207352
},
"reported": {
"count": 407,
"observable_mse": 0.000869595562107861,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04342574253678322
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006972837665095773
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0009436380350962281,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04636046662926674
},
"reported": {
"count": 448,
"observable_mse": 0.00094363815151155,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04636046662926674
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010637716654662544
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0009590117260813713,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046864788979291916
},
"reported": {
"count": 2281,
"observable_mse": 0.0009590117260813713,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046864788979291916
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011598924174904823
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010999653022736311,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010999666119460016,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00012954231484367922
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011603816412389278
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010919027408817783,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010919012856902555,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014394738889362153
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010782991535961628
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011893396003870293,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011893427290488034,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003091888621587732
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010790933854877949
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011383872333681211,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011383901437511668,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00028776244525315775
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03806070610880852
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00042644605855457485,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.000426446640631184,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005582693981920547
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0369470976293087
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0004731984226964414,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00047319853911176324,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011115546162157263
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.035910893231630325
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0003548956592567265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00035489487345330417,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.8580342233181e-10,
"numeric_tolerance_ratio": 0.0007588714631438345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020458154380321503
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018904275202658027,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018904254829976708,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.00019994696482166673
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 6.998432908560917e-16
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.00634012371301651
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00012711752788163722,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00012711742601823062,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.0001005847985299984
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008087845519185066
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.686173143563792e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.68617750913836e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3236944961253016e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.698037759633735e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.059283086545828e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012188127264380455
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00014413957251235843,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00014413951430469751,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.738058002000685e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252827632939443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.5279510989785194e-10,
"numeric_tolerance_ratio": 0.00015124442457133805
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011474297207314521,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011474302300484851,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.035392711752096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010750066576292738,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010750071669463068,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0390007111411554e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010249437036691234,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6010698552450364e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001481530343880877,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00014815300528425723,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8678943293010077e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 0.00010700838174670935,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 0.00010700869461288676,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003095536838477713
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 6.998753087827936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 6.998747994657606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.057772256490849e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.253204239532351e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.253222429426387e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-10,
"numeric_tolerance_ratio": 0.0001805890875343717
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002318347105756402,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00023183472512755543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.422219535332207e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.00010696651588659734,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 1.5491703379666433e-05,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 1.5491703379666433e-05
},
"gradients": {
"norms": {
"prediction": 0.0003186509129591286,
"weighted_latent": 2.6831793128829187e-18,
"total": 0.0003186509129591286
},
"prediction_weighted_latent_dot": 7.71858239031985e-23,
"prediction_weighted_latent_cosine": 0.09027607741696084,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "e10d1479ab2365c43ffe7e27c9b1efa8384babeabb8f6328e4de58db8d11fd16"
}
],
"final_losses": {
"prediction_mse": 0.00010696651588659734,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 1.5491703379666433e-05,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 1.5491703379666433e-05
},
"training_seconds": 4.3000036384910345e-05,
"evaluation_seconds": 0.6642293999902904
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 3,
"arm": "native",
"status": "passed",
"files": {
"seed-3-native/result.json": "c9dc68011f01b4a7571815fef2f6ed43b2845347c9e56f0a09546724685fa851",
"seed-3-native/checkpoint.json": "943e22bc3deb4e7c773e1efe266351336849e74d84ca86125d1aef1f2a7f1bf4",
"inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"seed-3-native/diagnostics-arrays.npz": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 5.811452865600586e-07,
"max_tolerance_ratio": 0.3990840958602454,
"worst_tolerance_index": [
60,
2
],
"actual_at_worst": 0.004562556743621826,
"saved_at_worst": 0.004561975598335266,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.00010696637036744505,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 0.00010696651588659734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014397905845926648
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001894764427561313,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 0.00018947666103485972,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-10,
"numeric_tolerance_ratio": 0.00021421976386698359
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2893109321594238,
"saved_at_worst": 1.2893109321594238,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.3867421263170503,
"worst_tolerance_index": [
77,
11
],
"actual_at_worst": 0.0007888078689575195,
"saved_at_worst": 0.0007883906364440918,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.0016188963782042265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16294221580028534
},
"reported": {
"count": 784,
"observable_mse": 0.0016188964946195483,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16294220089912415
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0008616266658095435
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 0.0017986265011131763,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.17537100613117218
},
"reported": {
"count": 580,
"observable_mse": 0.0017986266175284982,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.17537100613117218
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 9.866853626335093e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.0018708795541897416,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16294221580028534
},
"reported": {
"count": 112,
"observable_mse": 0.001870880019851029,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16294221580028534
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00039227191794461667
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 0.00017905373533722013,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"reported": {
"count": 92,
"observable_mse": 0.00017905370623338968,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8591882208963364e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.000102343074104283,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"reported": {
"count": 98,
"observable_mse": 0.00010234319051960483,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001757528884955003
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.67251874296926e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897133611142635
},
"reported": {
"count": 70,
"observable_mse": 8.67253402248025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0020504503657808367
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010630184988258407,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897137802094221
},
"reported": {
"count": 14,
"observable_mse": 0.00010630159522406757,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002343372258461114
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017647375352680683,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897135008126497
},
"reported": {
"count": 14,
"observable_mse": 0.00017647400090936571,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897139199078083
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002636292941319088
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252822539769113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.737066673347726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 70,
"observable_mse": 8.737093594390899e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010268489859299734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 14,
"observable_mse": 0.0001026846730383113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.255546860396862e-10,
"numeric_tolerance_ratio": 0.00022326212619665206
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017815936007536948,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"reported": {
"count": 14,
"observable_mse": 0.00017815965111367404,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002859439373556026
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001030568455462344,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580707758665
},
"reported": {
"count": 98,
"observable_mse": 0.00010305678733857349,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00579952893372632
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.592774247517809e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580242097378
},
"reported": {
"count": 70,
"observable_mse": 8.592770609539002e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585904866456985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.006379481642109011
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001099406072171405,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585797764360905
},
"reported": {
"count": 14,
"observable_mse": 0.00010994031617883593,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585867613554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.004349647583201486
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00018181865743827075,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058583967387676
},
"reported": {
"count": 14,
"observable_mse": 0.00018181874474976212,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003769693806922108
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200838732998818,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097275298088789
},
"reported": {
"count": 98,
"observable_mse": 0.00010200848191743717,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097279489040375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002603515450311345
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543613512301818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060906685888767
},
"reported": {
"count": 72,
"observable_mse": 8.543617150280625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060910876840353
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026094108969952367
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001222111313836649,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00609727343544364
},
"reported": {
"count": 14,
"observable_mse": 0.00012221158249303699,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060972776263952255
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026035157515688335
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00017787207616493106,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311086472123861
},
"reported": {
"count": 12,
"observable_mse": 0.00017787201795727015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311097647994757
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006851697621195759
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200543329119682,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094702053815126
},
"reported": {
"count": 98,
"observable_mse": 0.00010200551332673058,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094708573073149
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004050559842515715
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543937292415649e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058296654373407
},
"reported": {
"count": 72,
"observable_mse": 8.543945295969024e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058302707970142
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.0037697612537818566
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00012222511577419937,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060947067104279995
},
"reported": {
"count": 14,
"observable_mse": 0.00012222547957208008,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094712298363447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0034719076328323475
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.0001778121804818511,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006308719515800476
},
"reported": {
"count": 12,
"observable_mse": 0.0001778119767550379,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0063087306916713715
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006852692037580273
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067623384296894
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0023185078919724916
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.603723108535632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067622918635607
},
"reported": {
"count": 70,
"observable_mse": 8.603732567280531e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.00202869440547593
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010919466149061918,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067620124667883
},
"reported": {
"count": 14,
"observable_mse": 0.00010919448686763644,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028981346969819097
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017808005213737488,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067628972232342
},
"reported": {
"count": 14,
"observable_mse": 0.00017807971744332463,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.005216642454567437
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.012269754894077778,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055108308792114
},
"reported": {
"count": 98,
"observable_mse": 0.012269754894077778,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055108308792114
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.011943741701543331,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055108308792114
},
"reported": {
"count": 84,
"observable_mse": 0.011943741701543331,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055111289024353
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0006333493125675963
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.014225834980607033,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055111289024353
},
"reported": {
"count": 14,
"observable_mse": 0.014225835911929607,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.46055108308792114
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.000633349352680734
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.69803557684645e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705172687768936
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.005974748847249882
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 4.724697646452114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754040114581585
},
"reported": {
"count": 72,
"observable_mse": 4.724687460111454e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754054084420204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.007448969996752607
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.864315218990669e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 14,
"observable_mse": 6.864342140033841e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705178275704384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6763806343078613e-08,
"numeric_tolerance_ratio": 0.008962120593553839
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00018344067211728543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405925706028938
},
"reported": {
"count": 12,
"observable_mse": 0.00018344077398069203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405914530158043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.0060718911179741195
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869952917099,
"saved_at_worst": 1.2869952917099,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.47687637075463457,
"worst_tolerance_index": [
219,
2
],
"actual_at_worst": -0.0012496113777160645,
"saved_at_worst": -0.001249074935913086,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.00640147877857089,
"exact_state_accuracy": 0.9738520408163265,
"per_field_accuracy": {
"x": 0.9805484693877551,
"y": 0.9950573979591837,
"h": 0.9956951530612245
},
"normalized_latent_rmse": 0.2254096269607544
},
"reported": {
"count": 6272,
"observable_mse": 0.006401479244232178,
"exact_state_accuracy": 0.9738520383834839,
"per_field_accuracy": {
"x": 0.9805484414100647,
"y": 0.9950574040412903,
"h": 0.9956951141357422
},
"normalized_latent_rmse": 0.2254096269607544
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.892548228190407e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002839142008922737
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.007140567060559988,
"exact_state_accuracy": 0.9701857282502444,
"per_field_accuracy": {
"x": 0.9782502443792767,
"y": 0.9943792766373412,
"h": 0.9946236559139785
},
"normalized_latent_rmse": 0.23947079479694366
},
"reported": {
"count": 4092,
"observable_mse": 0.007140567060559988,
"exact_state_accuracy": 0.9701857566833496,
"per_field_accuracy": {
"x": 0.97825026512146,
"y": 0.9943792819976807,
"h": 0.9946236610412598
},
"normalized_latent_rmse": 0.23947079479694366
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.843310520361797e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.006372062023729086,
"exact_state_accuracy": 0.9720982142857143,
"per_field_accuracy": {
"x": 0.9799107142857143,
"y": 0.9944196428571429,
"h": 0.9944196428571429
},
"normalized_latent_rmse": 0.2254096269607544
},
"reported": {
"count": 896,
"observable_mse": 0.006372062489390373,
"exact_state_accuracy": 0.9720982313156128,
"per_field_accuracy": {
"x": 0.9799107313156128,
"y": 0.9944196939468384,
"h": 0.9944196939468384
},
"normalized_latent_rmse": 0.2254096269607544
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.1089695474182406e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028442432809519443
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.0040665967389941216,
"exact_state_accuracy": 0.9867601246105919,
"per_field_accuracy": {
"x": 0.9883177570093458,
"y": 0.9976635514018691,
"h": 1.0
},
"normalized_latent_rmse": 0.17313982546329498
},
"reported": {
"count": 1284,
"observable_mse": 0.004066596273332834,
"exact_state_accuracy": 0.9867600798606873,
"per_field_accuracy": {
"x": 0.9883177280426025,
"y": 0.9976634979248047,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.1731398105621338
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0008136494816779414
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.4800254106521606,
"saved_at_worst": 1.4800254106521606,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.364418029785156e-07,
"max_tolerance_ratio": 0.4255259867803609,
"worst_tolerance_index": [
2845,
7
],
"actual_at_worst": -0.00138140469789505,
"saved_at_worst": -0.00138092041015625,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.01766131818294525,
"exact_state_accuracy": 0.9158163265306123,
"per_field_accuracy": {
"x": 0.9371811224489796,
"y": 0.9620535714285714,
"h": 1.0
},
"normalized_latent_rmse": 0.30599245429039
},
"reported": {
"count": 3136,
"observable_mse": 0.017661316320300102,
"exact_state_accuracy": 0.9158163070678711,
"per_field_accuracy": {
"x": 0.9371811151504517,
"y": 0.9620535373687744,
"h": 1.0
},
"normalized_latent_rmse": 0.30599245429039
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.405979698278827e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006733754560566584
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.019254149869084358,
"exact_state_accuracy": 0.9064475347661188,
"per_field_accuracy": {
"x": 0.9292035398230089,
"y": 0.95448798988622,
"h": 1.0
},
"normalized_latent_rmse": 0.31791701912879944
},
"reported": {
"count": 1582,
"observable_mse": 0.019254149869084358,
"exact_state_accuracy": 0.9064475893974304,
"per_field_accuracy": {
"x": 0.9292035698890686,
"y": 0.9544880390167236,
"h": 1.0
},
"normalized_latent_rmse": 0.31791701912879944
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.463131158567336e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.017359385266900063,
"exact_state_accuracy": 0.9151785714285714,
"per_field_accuracy": {
"x": 0.9375,
"y": 0.9620535714285714,
"h": 1.0
},
"normalized_latent_rmse": 0.30599245429039
},
"reported": {
"count": 448,
"observable_mse": 0.017359385266900063,
"exact_state_accuracy": 0.9151785969734192,
"per_field_accuracy": {
"x": 0.9375000596046448,
"y": 0.9620535969734192,
"h": 1.0
},
"normalized_latent_rmse": 0.30599245429039
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.01550526637583971,
"exact_state_accuracy": 0.9294755877034359,
"per_field_accuracy": {
"x": 0.9484629294755877,
"y": 0.972875226039783,
"h": 1.0
},
"normalized_latent_rmse": 0.28807908296585083
},
"reported": {
"count": 1106,
"observable_mse": 0.01550526823848486,
"exact_state_accuracy": 0.9294756054878235,
"per_field_accuracy": {
"x": 0.948462963104248,
"y": 0.9728752374649048,
"h": 1.0
},
"normalized_latent_rmse": 0.28807908296585083
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.36286603008773e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0007302982002833496
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.515133023262024,
"saved_at_worst": 1.515133023262024,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.35522893439134723,
"worst_tolerance_index": [
716,
3
],
"actual_at_worst": 6.788969039916992e-05,
"saved_at_worst": 6.753206253051758e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.044902168214321136,
"exact_state_accuracy": 0.7213010204081632,
"per_field_accuracy": {
"x": 0.8118622448979592,
"y": 0.9298469387755102,
"h": 0.9371811224489796
},
"normalized_latent_rmse": 0.4604598581790924
},
"reported": {
"count": 3136,
"observable_mse": 0.044902168214321136,
"exact_state_accuracy": 0.7213010191917419,
"per_field_accuracy": {
"x": 0.8118622303009033,
"y": 0.9298469424247742,
"h": 0.9371811151504517
},
"normalized_latent_rmse": 0.4604598581790924
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.4597055897347389e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.048988405615091324,
"exact_state_accuracy": 0.6993006993006993,
"per_field_accuracy": {
"x": 0.8001998001998002,
"y": 0.9430569430569431,
"h": 0.9140859140859141
},
"normalized_latent_rmse": 0.4937666654586792
},
"reported": {
"count": 1001,
"observable_mse": 0.048988401889801025,
"exact_state_accuracy": 0.6993007063865662,
"per_field_accuracy": {
"x": 0.8001998066902161,
"y": 0.9430569410324097,
"h": 0.914085865020752
},
"normalized_latent_rmse": 0.4937666654586792
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.906516215541501e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006315292801831286
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.044932298362255096,
"exact_state_accuracy": 0.7254464285714286,
"per_field_accuracy": {
"x": 0.8214285714285714,
"y": 0.9330357142857143,
"h": 0.9375
},
"normalized_latent_rmse": 0.4604598879814148
},
"reported": {
"count": 448,
"observable_mse": 0.044932298362255096,
"exact_state_accuracy": 0.7254464626312256,
"per_field_accuracy": {
"x": 0.8214285969734192,
"y": 0.9330357313156128,
"h": 0.9375000596046448
},
"normalized_latent_rmse": 0.4604598581790924
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0006334721628120355
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.04246954992413521,
"exact_state_accuracy": 0.7332542975696502,
"per_field_accuracy": {
"x": 0.8162418494368702,
"y": 0.921161825726141,
"h": 0.950800237107291
},
"normalized_latent_rmse": 0.43950536847114563
},
"reported": {
"count": 1687,
"observable_mse": 0.04246955364942551,
"exact_state_accuracy": 0.7332543134689331,
"per_field_accuracy": {
"x": 0.8162418603897095,
"y": 0.9211618900299072,
"h": 0.9508002996444702
},
"normalized_latent_rmse": 0.43950533866882324
},
"differences": [],
"max_errors": {
"accuracy_abs": 6.430376620070177e-08,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0007099908498083255
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.4931187629699707,
"saved_at_worst": 1.4931187629699707,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.38143623930895526,
"worst_tolerance_index": [
2582,
8
],
"actual_at_worst": -0.0009388774633407593,
"saved_at_worst": -0.0009384602308273315,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.08657023310661316,
"exact_state_accuracy": 0.5446428571428571,
"per_field_accuracy": {
"x": 0.7085459183673469,
"y": 0.7911352040816326,
"h": 0.8590561224489796
},
"normalized_latent_rmse": 0.5297196507453918
},
"reported": {
"count": 3136,
"observable_mse": 0.08657022565603256,
"exact_state_accuracy": 0.5446428656578064,
"per_field_accuracy": {
"x": 0.7085459232330322,
"y": 0.7911351919174194,
"h": 0.8590561151504517
},
"normalized_latent_rmse": 0.5297196507453918
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.216421319227834e-08,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0007715194353445527
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.11863617599010468,
"exact_state_accuracy": 0.40540540540540543,
"per_field_accuracy": {
"x": 0.6486486486486487,
"y": 0.687960687960688,
"h": 0.7985257985257985
},
"normalized_latent_rmse": 0.6136864423751831
},
"reported": {
"count": 407,
"observable_mse": 0.11863616108894348,
"exact_state_accuracy": 0.4054054021835327,
"per_field_accuracy": {
"x": 0.6486486196517944,
"y": 0.687960684299469,
"h": 0.7985258102416992
},
"normalized_latent_rmse": 0.6136864423751831
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.8996854251062132e-08,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.001158395980391896
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.08620558679103851,
"exact_state_accuracy": 0.5379464285714286,
"per_field_accuracy": {
"x": 0.703125,
"y": 0.7946428571428571,
"h": 0.8616071428571429
},
"normalized_latent_rmse": 0.5297196507453918
},
"reported": {
"count": 448,
"observable_mse": 0.08620557934045792,
"exact_state_accuracy": 0.5379464626312256,
"per_field_accuracy": {
"x": 0.7031250596046448,
"y": 0.7946428656578064,
"h": 0.8616071939468384
},
"normalized_latent_rmse": 0.5297196507453918
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0007744437118929744
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.08092029392719269,
"exact_state_accuracy": 0.5708022797018851,
"per_field_accuracy": {
"x": 0.7202981148619026,
"y": 0.8088557650153442,
"h": 0.8693555458132398
},
"normalized_latent_rmse": 0.5132953524589539
},
"reported": {
"count": 2281,
"observable_mse": 0.08092029392719269,
"exact_state_accuracy": 0.5708022713661194,
"per_field_accuracy": {
"x": 0.7202981114387512,
"y": 0.8088557720184326,
"h": 0.8693555593490601
},
"normalized_latent_rmse": 0.5132953524589539
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.353582024243849e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011598924174904823
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010999653022736311,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010999666119460016,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00012954231484367922
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011603816412389278
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010919027408817783,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010919012856902555,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014394738889362153
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010782991535961628
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011893396003870293,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011893427290488034,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003091888621587732
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010790933854877949
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011383872333681211,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011383901437511668,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00028776244525315775
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03806070610880852
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00042644605855457485,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.000426446640631184,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005582693981920547
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0369470976293087
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0004731984226964414,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00047319853911176324,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011115546162157263
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.035910893231630325
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0003548956592567265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00035489487345330417,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.8580342233181e-10,
"numeric_tolerance_ratio": 0.0007588714631438345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020458154380321503
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018904275202658027,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018904254829976708,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.00019994696482166673
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 0.6938775510204082,
"normalized_latent_relation_rmse": 0.5547308325767517
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.04983695596456528,
"exact_state_accuracy": 0.6938775510204082,
"per_field_accuracy": {
"x": 0.9183673469387755,
"y": 1.0,
"h": 0.7244897959183674
}
},
"reported": {
"count": 98,
"observable_mse": 0.04983697086572647,
"exact_state_accuracy": 0.6938775181770325,
"per_field_accuracy": {
"x": 0.918367326259613,
"y": 1.0,
"h": 0.7244898080825806
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.28433756857649e-08,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.002490293371849605
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 0.20408163265306123,
"normalized_latent_relation_rmse": 0.6198129653930664
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.11789534240961075,
"exact_state_accuracy": 0.20408163265306123,
"per_field_accuracy": {
"x": 0.2755102040816326,
"y": 0.9795918367346939,
"h": 0.8163265306122449
}
},
"reported": {
"count": 98,
"observable_mse": 0.11789533495903015,
"exact_state_accuracy": 0.20408162474632263,
"per_field_accuracy": {
"x": 0.27551019191741943,
"y": 0.9795918464660645,
"h": 0.8163264989852905
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.1626954388741524e-08,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0005825529601459301
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008087845519185066
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.686173143563792e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.68617750913836e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3236944961253016e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.698037759633735e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.059283086545828e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012188127264380455
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00014413957251235843,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00014413951430469751,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.738058002000685e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252827632939443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.5279510989785194e-10,
"numeric_tolerance_ratio": 0.00015124442457133805
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011474297207314521,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011474302300484851,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.035392711752096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010750066576292738,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010750071669463068,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0390007111411554e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 0.7653061224489796
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0509251169860363,
"exact_state_accuracy": 0.6224489795918368,
"per_field_accuracy": {
"x": 0.6428571428571429,
"y": 0.9693877551020408,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0509251169860363,
"exact_state_accuracy": 0.6224489808082581,
"per_field_accuracy": {
"x": 0.6428571343421936,
"y": 0.9693877100944519,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.5007588878043236e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.012192781083285809,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.01219277922064066,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0008393023382572028
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 0.00010700838174670935,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 0.00010700869461288676,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003095536838477713
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 6.998753087827936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 6.998747994657606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.057772256490849e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.253204239532351e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.253222429426387e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-10,
"numeric_tolerance_ratio": 0.0001805890875343717
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002318347105756402,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00023183472512755543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.422219535332207e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 774,
"error_count": 206,
"per_field_correct_counts": {
"x": 811,
"y": 889,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 210,
"error_count": 476,
"per_field_correct_counts": {
"x": 450,
"y": 482,
"h": 515
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 212,
"error_count": 278,
"per_field_correct_counts": {
"x": 269,
"y": 401,
"h": 453
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 0,
"error_count": 98,
"per_field_correct_counts": {
"x": 98,
"y": 21,
"h": 49
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 4,
"error_count": 192,
"per_field_correct_counts": {
"x": 14,
"y": 143,
"h": 104
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 0,
"error_count": 98,
"per_field_correct_counts": {
"x": 0,
"y": 21,
"h": 54
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 18,
"error_count": 80,
"per_field_correct_counts": {
"x": 90,
"y": 34,
"h": 49
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 1708,
"accuracy": 0.5446428571428571
},
"numpy": {
"cases": 3136,
"correct": 1708,
"accuracy": 0.5446428571428571
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.011943741701543331,
"latent_mse": 0.212107315659523,
"scaled_prediction": 0.0017297833692282438,
"scaled_weighted_latent": 0.0030718990601599216,
"total": 0.004801682196557522
},
"gradients": {
"norms": {
"prediction": 0.0034363465383648872,
"weighted_latent": 0.004243341740220785,
"total": 0.0022537540644407272
},
"prediction_weighted_latent_dot": -1.2367510862532072e-05,
"prediction_weighted_latent_cosine": -0.8481591220444237,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "943e22bc3deb4e7c773e1efe266351336849e74d84ca86125d1aef1f2a7f1bf4"
}
],
"final_losses": {
"prediction_mse": 0.011943741701543331,
"latent_mse": 0.212107315659523,
"scaled_prediction": 0.0017297833692282438,
"scaled_weighted_latent": 0.0030718990601599216,
"total": 0.004801682196557522
},
"training_seconds": 4.0899962186813354e-05,
"evaluation_seconds": 0.5442906999960542
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 3,
"arm": "prediction_latent",
"status": "passed",
"files": {
"seed-3-prediction_latent/result.json": "b2c4aa384a0aeec0941d8e7ecbcf32ca812b4066261f9896fde0676749ca3402",
"seed-3-prediction_latent/checkpoint.json": "fe943a3c3a39ef1a0d8246e36a9c3703472e67a373e38cf26ebb5945840de54e",
"inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"seed-3-prediction_latent/diagnostics-arrays.npz": "c76bf30e8ceb95e67340c590a1f774ac16bb2f18e84ab8870d9b8e3fa8d3fdc0"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 5.811452865600586e-07,
"max_tolerance_ratio": 0.3990840958602454,
"worst_tolerance_index": [
60,
2
],
"actual_at_worst": 0.004562556743621826,
"saved_at_worst": 0.004561975598335266,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.00010696637036744505,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 0.00010696651588659734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014397905845926648
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001894764427561313,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 0.00018947666103485972,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-10,
"numeric_tolerance_ratio": 0.00021421976386698359
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2893109321594238,
"saved_at_worst": 1.2893109321594238,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.3867421263170503,
"worst_tolerance_index": [
77,
11
],
"actual_at_worst": 0.0007888078689575195,
"saved_at_worst": 0.0007883906364440918,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.0012843458680436015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.11893079429864883
},
"reported": {
"count": 784,
"observable_mse": 0.0012843457516282797,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.11893079429864883
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010316532689556824
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 0.0014315596781671047,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1279904842376709
},
"reported": {
"count": 580,
"observable_mse": 0.0014315597945824265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.1279904842376709
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010183677811150988
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.001429907395504415,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.11893079429864883
},
"reported": {
"count": 112,
"observable_mse": 0.001429907395504415,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.11893079429864883
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 0.00017905373533722013,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"reported": {
"count": 92,
"observable_mse": 0.00017905370623338968,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8591882208963364e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.000102343074104283,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"reported": {
"count": 98,
"observable_mse": 0.00010234319051960483,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001757528884955003
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.67251874296926e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897133611142635
},
"reported": {
"count": 70,
"observable_mse": 8.67253402248025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0020504503657808367
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010630184988258407,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897137802094221
},
"reported": {
"count": 14,
"observable_mse": 0.00010630159522406757,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002343372258461114
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017647375352680683,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897135008126497
},
"reported": {
"count": 14,
"observable_mse": 0.00017647400090936571,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897139199078083
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002636292941319088
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252822539769113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.737066673347726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 70,
"observable_mse": 8.737093594390899e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010268489859299734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 14,
"observable_mse": 0.0001026846730383113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.255546860396862e-10,
"numeric_tolerance_ratio": 0.00022326212619665206
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017815936007536948,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"reported": {
"count": 14,
"observable_mse": 0.00017815965111367404,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002859439373556026
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001030568455462344,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580707758665
},
"reported": {
"count": 98,
"observable_mse": 0.00010305678733857349,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00579952893372632
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.592774247517809e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580242097378
},
"reported": {
"count": 70,
"observable_mse": 8.592770609539002e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585904866456985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.006379481642109011
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001099406072171405,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585797764360905
},
"reported": {
"count": 14,
"observable_mse": 0.00010994031617883593,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585867613554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.004349647583201486
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00018181865743827075,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058583967387676
},
"reported": {
"count": 14,
"observable_mse": 0.00018181874474976212,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003769693806922108
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200838732998818,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097275298088789
},
"reported": {
"count": 98,
"observable_mse": 0.00010200848191743717,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097279489040375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002603515450311345
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543613512301818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060906685888767
},
"reported": {
"count": 72,
"observable_mse": 8.543617150280625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060910876840353
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026094108969952367
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001222111313836649,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00609727343544364
},
"reported": {
"count": 14,
"observable_mse": 0.00012221158249303699,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060972776263952255
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026035157515688335
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00017787207616493106,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311086472123861
},
"reported": {
"count": 12,
"observable_mse": 0.00017787201795727015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311097647994757
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006851697621195759
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200543329119682,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094702053815126
},
"reported": {
"count": 98,
"observable_mse": 0.00010200551332673058,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094708573073149
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004050559842515715
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543937292415649e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058296654373407
},
"reported": {
"count": 72,
"observable_mse": 8.543945295969024e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058302707970142
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.0037697612537818566
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00012222511577419937,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060947067104279995
},
"reported": {
"count": 14,
"observable_mse": 0.00012222547957208008,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094712298363447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0034719076328323475
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.0001778121804818511,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006308719515800476
},
"reported": {
"count": 12,
"observable_mse": 0.0001778119767550379,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0063087306916713715
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006852692037580273
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067623384296894
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0023185078919724916
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.603723108535632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067622918635607
},
"reported": {
"count": 70,
"observable_mse": 8.603732567280531e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.00202869440547593
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010919466149061918,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067620124667883
},
"reported": {
"count": 14,
"observable_mse": 0.00010919448686763644,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028981346969819097
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017808005213737488,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067628972232342
},
"reported": {
"count": 14,
"observable_mse": 0.00017807971744332463,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.005216642454567437
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.009593349881470203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594977855682373
},
"reported": {
"count": 98,
"observable_mse": 0.009593350812792778,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594974875450134
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9802322387695312e-08,
"numeric_tolerance_ratio": 0.0008614639118828826
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.009409232065081596,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594974875450134
},
"reported": {
"count": 84,
"observable_mse": 0.009409232996404171,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594974875450134
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004798348161351965
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.010698058642446995,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594974875450134
},
"reported": {
"count": 14,
"observable_mse": 0.01069805957376957,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.33594974875450134
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00044995646635191525
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.69803557684645e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705172687768936
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.005974748847249882
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 4.724697646452114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754040114581585
},
"reported": {
"count": 72,
"observable_mse": 4.724687460111454e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754054084420204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.007448969996752607
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.864315218990669e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 14,
"observable_mse": 6.864342140033841e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705178275704384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6763806343078613e-08,
"numeric_tolerance_ratio": 0.008962120593553839
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00018344067211728543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405925706028938
},
"reported": {
"count": 12,
"observable_mse": 0.00018344077398069203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405914530158043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.0060718911179741195
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869952917099,
"saved_at_worst": 1.2869952917099,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.5021484272770335,
"worst_tolerance_index": [
5148,
2
],
"actual_at_worst": -8.893013000488281e-05,
"saved_at_worst": -8.943676948547363e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.004743542522192001,
"exact_state_accuracy": 0.9770408163265306,
"per_field_accuracy": {
"x": 0.9858099489795918,
"y": 0.9912308673469388,
"h": 1.0
},
"normalized_latent_rmse": 0.1644071638584137
},
"reported": {
"count": 6272,
"observable_mse": 0.004743542056530714,
"exact_state_accuracy": 0.9770407676696777,
"per_field_accuracy": {
"x": 0.9858099222183228,
"y": 0.991230845451355,
"h": 1.0
},
"normalized_latent_rmse": 0.1644071638584137
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.865685288013566e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00031584085121626025
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.005354098044335842,
"exact_state_accuracy": 0.9733626588465298,
"per_field_accuracy": {
"x": 0.9838709677419355,
"y": 0.9894916911045943,
"h": 1.0
},
"normalized_latent_rmse": 0.17495006322860718
},
"reported": {
"count": 4092,
"observable_mse": 0.005354098044335842,
"exact_state_accuracy": 0.9733626842498779,
"per_field_accuracy": {
"x": 0.9838709831237793,
"y": 0.9894917011260986,
"h": 1.0
},
"normalized_latent_rmse": 0.17495006322860718
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.540334809175704e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.004613194614648819,
"exact_state_accuracy": 0.9776785714285714,
"per_field_accuracy": {
"x": 0.9866071428571429,
"y": 0.9910714285714286,
"h": 1.0
},
"normalized_latent_rmse": 0.1644071638584137
},
"reported": {
"count": 896,
"observable_mse": 0.004613194614648819,
"exact_state_accuracy": 0.9776785969734192,
"per_field_accuracy": {
"x": 0.9866071939468384,
"y": 0.9910714626312256,
"h": 1.0
},
"normalized_latent_rmse": 0.1644071638584137
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.1089695474182406e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.002888710703700781,
"exact_state_accuracy": 0.9883177570093458,
"per_field_accuracy": {
"x": 0.9914330218068536,
"y": 0.9968847352024922,
"h": 1.0
},
"normalized_latent_rmse": 0.12501011788845062
},
"reported": {
"count": 1284,
"observable_mse": 0.0028887102380394936,
"exact_state_accuracy": 0.9883177280426025,
"per_field_accuracy": {
"x": 0.9914329648017883,
"y": 0.9968847036361694,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.12501011788845062
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00036129393764582855
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.280820369720459,
"saved_at_worst": 1.280820369720459,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.5498198938627676,
"worst_tolerance_index": [
2107,
2
],
"actual_at_worst": 2.8252601623535156e-05,
"saved_at_worst": 2.7701258659362793e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.01111413910984993,
"exact_state_accuracy": 0.9346301020408163,
"per_field_accuracy": {
"x": 0.9764030612244898,
"y": 0.9566326530612245,
"h": 1.0
},
"normalized_latent_rmse": 0.21937502920627594
},
"reported": {
"count": 3136,
"observable_mse": 0.01111413724720478,
"exact_state_accuracy": 0.9346300959587097,
"per_field_accuracy": {
"x": 0.9764030575752258,
"y": 0.9566326141357422,
"h": 1.0
},
"normalized_latent_rmse": 0.21937502920627594
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.892548228190407e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0008821791425446691
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.012310564517974854,
"exact_state_accuracy": 0.9254108723135271,
"per_field_accuracy": {
"x": 0.9740834386852086,
"y": 0.9487989886219975,
"h": 1.0
},
"normalized_latent_rmse": 0.2291337251663208
},
"reported": {
"count": 1582,
"observable_mse": 0.012310565449297428,
"exact_state_accuracy": 0.9254109263420105,
"per_field_accuracy": {
"x": 0.9740834832191467,
"y": 0.9487990140914917,
"h": 1.0
},
"normalized_latent_rmse": 0.2291337251663208
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.4028483353896206e-08,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00041743566595476244
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.011239134706556797,
"exact_state_accuracy": 0.9397321428571429,
"per_field_accuracy": {
"x": 0.9776785714285714,
"y": 0.9598214285714286,
"h": 1.0
},
"normalized_latent_rmse": 0.21937502920627594
},
"reported": {
"count": 448,
"observable_mse": 0.011239135637879372,
"exact_state_accuracy": 0.9397321939468384,
"per_field_accuracy": {
"x": 0.9776785969734192,
"y": 0.9598214626312256,
"h": 1.0
},
"normalized_latent_rmse": 0.21937502920627594
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.1089695474182406e-08,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00043849363293037785
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.009352161549031734,
"exact_state_accuracy": 0.945750452079566,
"per_field_accuracy": {
"x": 0.9792043399638336,
"y": 0.9665461121157324,
"h": 1.0
},
"normalized_latent_rmse": 0.2046089768409729
},
"reported": {
"count": 1106,
"observable_mse": 0.009352161549031734,
"exact_state_accuracy": 0.9457504749298096,
"per_field_accuracy": {
"x": 0.9792043566703796,
"y": 0.9665461182594299,
"h": 1.0
},
"normalized_latent_rmse": 0.2046089917421341
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.2850243586169938e-08,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.000694340021491379
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.363230586051941,
"saved_at_worst": 1.363230586051941,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.556510925292969e-07,
"max_tolerance_ratio": 0.43877816078235904,
"worst_tolerance_index": [
174,
6
],
"actual_at_worst": -0.000697113573551178,
"saved_at_worst": -0.0006975829601287842,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.031687669456005096,
"exact_state_accuracy": 0.7955994897959183,
"per_field_accuracy": {
"x": 0.8931760204081632,
"y": 0.9024234693877551,
"h": 1.0
},
"normalized_latent_rmse": 0.3262229263782501
},
"reported": {
"count": 3136,
"observable_mse": 0.031687669456005096,
"exact_state_accuracy": 0.7955994606018066,
"per_field_accuracy": {
"x": 0.8931760191917419,
"y": 0.9024234414100647,
"h": 1.0
},
"normalized_latent_rmse": 0.3262229263782501
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.9194111683672475e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.03607439249753952,
"exact_state_accuracy": 0.7812187812187812,
"per_field_accuracy": {
"x": 0.8851148851148851,
"y": 0.8961038961038961,
"h": 1.0
},
"normalized_latent_rmse": 0.3467058539390564
},
"reported": {
"count": 1001,
"observable_mse": 0.03607438877224922,
"exact_state_accuracy": 0.7812187671661377,
"per_field_accuracy": {
"x": 0.885114848613739,
"y": 0.8961038589477539,
"h": 1.0
},
"normalized_latent_rmse": 0.3467058539390564
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.715614216304175e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0008085381917655889
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.03193138167262077,
"exact_state_accuracy": 0.7924107142857143,
"per_field_accuracy": {
"x": 0.8950892857142857,
"y": 0.8973214285714286,
"h": 1.0
},
"normalized_latent_rmse": 0.3262229263782501
},
"reported": {
"count": 448,
"observable_mse": 0.03193138539791107,
"exact_state_accuracy": 0.7924107313156128,
"per_field_accuracy": {
"x": 0.895089328289032,
"y": 0.8973214626312256,
"h": 1.0
},
"normalized_latent_rmse": 0.3262229263782501
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.257474628399649e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.000888425284094596
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.029020041227340698,
"exact_state_accuracy": 0.8049792531120332,
"per_field_accuracy": {
"x": 0.8974510966212211,
"y": 0.9075281564908121,
"h": 1.0
},
"normalized_latent_rmse": 0.3134370446205139
},
"reported": {
"count": 1687,
"observable_mse": 0.029020043089985847,
"exact_state_accuracy": 0.8049792647361755,
"per_field_accuracy": {
"x": 0.8974511027336121,
"y": 0.9075281620025635,
"h": 1.0
},
"normalized_latent_rmse": 0.3134370446205139
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.1624142315369568e-08,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0004773559949524988
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.340502381324768,
"saved_at_worst": 1.340502381324768,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.3502697441310343,
"worst_tolerance_index": [
161,
2
],
"actual_at_worst": 0.0006358623504638672,
"saved_at_worst": 0.000635489821434021,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.046413373202085495,
"exact_state_accuracy": 0.6910076530612245,
"per_field_accuracy": {
"x": 0.857780612244898,
"y": 0.7991071428571429,
"h": 0.9971301020408163
},
"normalized_latent_rmse": 0.3625972270965576
},
"reported": {
"count": 3136,
"observable_mse": 0.046413373202085495,
"exact_state_accuracy": 0.6910076141357422,
"per_field_accuracy": {
"x": 0.8577805757522583,
"y": 0.7991071343421936,
"h": 0.9971300959587097
},
"normalized_latent_rmse": 0.3625972270965576
},
"differences": [],
"max_errors": {
"accuracy_abs": 3.892548228190407e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.06509196758270264,
"exact_state_accuracy": 0.5454545454545454,
"per_field_accuracy": {
"x": 0.7936117936117936,
"y": 0.687960687960688,
"h": 0.995085995085995
},
"normalized_latent_rmse": 0.41973230242729187
},
"reported": {
"count": 407,
"observable_mse": 0.06509196013212204,
"exact_state_accuracy": 0.5454545617103577,
"per_field_accuracy": {
"x": 0.7936117649078369,
"y": 0.687960684299469,
"h": 0.9950859546661377
},
"normalized_latent_rmse": 0.41973230242729187
},
"differences": [],
"max_errors": {
"accuracy_abs": 4.041985735003806e-08,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0009921941821487623
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.04654546454548836,
"exact_state_accuracy": 0.6852678571428571,
"per_field_accuracy": {
"x": 0.8482142857142857,
"y": 0.7991071428571429,
"h": 1.0
},
"normalized_latent_rmse": 0.36259725689888
},
"reported": {
"count": 448,
"observable_mse": 0.04654546454548836,
"exact_state_accuracy": 0.6852678656578064,
"per_field_accuracy": {
"x": 0.848214328289032,
"y": 0.7991071939468384,
"h": 1.0
},
"normalized_latent_rmse": 0.36259725689888
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.1089695474182406e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.04305460304021835,
"exact_state_accuracy": 0.7181060938185007,
"per_field_accuracy": {
"x": 0.8711091626479615,
"y": 0.8189390618149934,
"h": 0.9969311705392372
},
"normalized_latent_rmse": 0.3514273464679718
},
"reported": {
"count": 2281,
"observable_mse": 0.04305460304021835,
"exact_state_accuracy": 0.7181060910224915,
"per_field_accuracy": {
"x": 0.8711091876029968,
"y": 0.8189390897750854,
"h": 0.9969311952590942
},
"normalized_latent_rmse": 0.3514273464679718
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.7960092019441163e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011598924174904823
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010999653022736311,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010999666119460016,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00012954231484367922
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011603816412389278
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010919027408817783,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010919012856902555,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014394738889362153
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010782991535961628
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011893396003870293,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011893427290488034,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003091888621587732
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010790933854877949
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011383872333681211,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011383901437511668,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00028776244525315775
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03806070610880852
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00042644605855457485,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.000426446640631184,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005582693981920547
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0369470976293087
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0004731984226964414,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00047319853911176324,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011115546162157263
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.035910893231630325
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0003548956592567265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00035489487345330417,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.8580342233181e-10,
"numeric_tolerance_ratio": 0.0007588714631438345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020458154380321503
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018904275202658027,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018904254829976708,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.00019994696482166673
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 0.8775510204081632,
"normalized_latent_relation_rmse": 0.37731829285621643
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.02961549162864685,
"exact_state_accuracy": 0.8775510204081632,
"per_field_accuracy": {
"x": 0.8775510204081632,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0296154897660017,
"exact_state_accuracy": 0.8775510191917419,
"per_field_accuracy": {
"x": 0.8775510191917419,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.2164212970233734e-09,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.00047018102268408466
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 0.41836734693877553,
"normalized_latent_relation_rmse": 0.4238298237323761
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.08056417107582092,
"exact_state_accuracy": 0.41836734693877553,
"per_field_accuracy": {
"x": 0.41836734693877553,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.08056416362524033,
"exact_state_accuracy": 0.41836732625961304,
"per_field_accuracy": {
"x": 0.41836732625961304,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.0679162493486558e-08,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.000822685298321172
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008087845519185066
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.686173143563792e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.68617750913836e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3236944961253016e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.698037759633735e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.059283086545828e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012188127264380455
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00014413957251235843,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00014413951430469751,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.738058002000685e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252827632939443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.5279510989785194e-10,
"numeric_tolerance_ratio": 0.00015124442457133805
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011474297207314521,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011474302300484851,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.035392711752096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010750066576292738,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010750071669463068,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0390007111411554e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 0.7857142857142857
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.04079073667526245,
"exact_state_accuracy": 0.6632653061224489,
"per_field_accuracy": {
"x": 0.6632653061224489,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.04079073294997215,
"exact_state_accuracy": 0.6632652878761292,
"per_field_accuracy": {
"x": 0.6632652878761292,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 1.824631978841751e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0007334586610772579
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.009554969146847725,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00955496821552515,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00047625880254618854
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 0.00010700838174670935,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 0.00010700869461288676,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003095536838477713
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 6.998753087827936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 6.998747994657606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.057772256490849e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.253204239532351e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.253222429426387e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-10,
"numeric_tolerance_ratio": 0.0001805890875343717
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002318347105756402,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00023183472512755543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.422219535332207e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 861,
"error_count": 119,
"per_field_correct_counts": {
"x": 941,
"y": 899,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 382,
"error_count": 304,
"per_field_correct_counts": {
"x": 556,
"y": 512,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 314,
"error_count": 176,
"per_field_correct_counts": {
"x": 376,
"y": 414,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 21,
"error_count": 77,
"per_field_correct_counts": {
"x": 63,
"y": 28,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 55,
"error_count": 141,
"per_field_correct_counts": {
"x": 87,
"y": 114,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 25,
"error_count": 73,
"per_field_correct_counts": {
"x": 94,
"y": 28,
"h": 95
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 19,
"error_count": 79,
"per_field_correct_counts": {
"x": 83,
"y": 21,
"h": 92
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 2167,
"accuracy": 0.6910076530612245
},
"numpy": {
"cases": 3136,
"correct": 2167,
"accuracy": 0.6910076530612245
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.011943741701543331,
"latent_mse": 0.212107315659523,
"scaled_prediction": 0.0017297833692282438,
"scaled_weighted_latent": 0.0030718990601599216,
"total": 0.004801682196557522
},
"gradients": {
"norms": {
"prediction": 0.0034363465383648872,
"weighted_latent": 0.004243341740220785,
"total": 0.0022537540644407272
},
"prediction_weighted_latent_dot": -1.2367510862532072e-05,
"prediction_weighted_latent_cosine": -0.8481591220444237,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.012193200178444386,
"latent_mse": 0.20949852466583252,
"scaled_prediction": 0.0017659118166193366,
"scaled_weighted_latent": 0.0030341167002916336,
"total": 0.004800028633326292
},
"gradients": {
"norms": {
"prediction": 0.0036069219931960106,
"weighted_latent": 0.004211698193103075,
"total": 0.0025878201704472303
},
"prediction_weighted_latent_dot": -1.2025736396026332e-05,
"prediction_weighted_latent_cosine": -0.791621694330808,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.03035777248442173
},
{
"step": 100,
"losses": {
"prediction_mse": 0.012208578176796436,
"latent_mse": 0.12798641622066498,
"scaled_prediction": 0.0017681389581412077,
"scaled_weighted_latent": 0.001853596419095993,
"total": 0.0036217353772372007
},
"gradients": {
"norms": {
"prediction": 0.003790223505347967,
"weighted_latent": 0.003106442280113697,
"total": 0.0034755005035549402
},
"prediction_weighted_latent_dot": -5.968337063677609e-06,
"prediction_weighted_latent_cosine": -0.5069034335374009,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.023319583386182785
},
{
"step": 200,
"losses": {
"prediction_mse": 0.009409403428435326,
"latent_mse": 0.1128612533211708,
"scaled_prediction": 0.0013627412263303995,
"scaled_weighted_latent": 0.0016345423646271229,
"total": 0.0029972835909575224
},
"gradients": {
"norms": {
"prediction": 0.002912612399086356,
"weighted_latent": 0.002912527183070779,
"total": 1.4770042071177159e-05
},
"prediction_weighted_latent_dot": -8.482954399369191e-06,
"prediction_weighted_latent_cosine": -0.999987223162017,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00015583900676574558
},
{
"step": 300,
"losses": {
"prediction_mse": 0.009409047663211823,
"latent_mse": 0.11286406964063644,
"scaled_prediction": 0.0013626896543428302,
"scaled_weighted_latent": 0.0016345831099897623,
"total": 0.0029972726479172707
},
"gradients": {
"norms": {
"prediction": 0.0029125649016350508,
"weighted_latent": 0.0029125751461833715,
"total": 1.37335703698227e-07
},
"prediction_weighted_latent_dot": -8.483063538733404e-06,
"prediction_weighted_latent_cosine": -0.9999999286325258,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.854749484664353e-06
},
{
"step": 400,
"losses": {
"prediction_mse": 0.009409072808921337,
"latent_mse": 0.11286382377147675,
"scaled_prediction": 0.0013626933796331286,
"scaled_weighted_latent": 0.0016345795011147857,
"total": 0.0029972728807479143
},
"gradients": {
"norms": {
"prediction": 0.0029125979635864496,
"weighted_latent": 0.002912567462772131,
"total": 5.532348268388887e-07
},
"prediction_weighted_latent_dot": -8.483139026793651e-06,
"prediction_weighted_latent_cosine": -1.0,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.552038338559214e-05
},
{
"step": 500,
"losses": {
"prediction_mse": 0.009409597143530846,
"latent_mse": 0.11285925656557083,
"scaled_prediction": 0.0013627692824229598,
"scaled_weighted_latent": 0.001634513377211988,
"total": 0.0029972826596349478
},
"gradients": {
"norms": {
"prediction": 0.0029151239432394505,
"weighted_latent": 0.00291238259524107,
"total": 3.6262848880141973e-05
},
"prediction_weighted_latent_dot": -8.489301762892865e-06,
"prediction_weighted_latent_cosine": -0.9999229121623138,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.001324942335486412
},
{
"step": 600,
"losses": {
"prediction_mse": 0.009409203194081783,
"latent_mse": 0.11286260187625885,
"scaled_prediction": 0.0013627122389152646,
"scaled_weighted_latent": 0.001634561805985868,
"total": 0.0029972740449011326
},
"gradients": {
"norms": {
"prediction": 0.002913573756814003,
"weighted_latent": 0.0029125947039574385,
"total": 1.4871639905322809e-05
},
"prediction_weighted_latent_dot": -8.48594936542213e-06,
"prediction_weighted_latent_cosine": -0.9999870224497254,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0002832206664606929
},
{
"step": 700,
"losses": {
"prediction_mse": 0.009409106336534023,
"latent_mse": 0.1128636971116066,
"scaled_prediction": 0.0013626981526613235,
"scaled_weighted_latent": 0.0016345777548849583,
"total": 0.002997275907546282
},
"gradients": {
"norms": {
"prediction": 0.0029136694502085447,
"weighted_latent": 0.0029126203153282404,
"total": 2.4933979148045182e-05
},
"prediction_weighted_latent_dot": -8.48610307002673e-06,
"prediction_weighted_latent_cosine": -0.9999634989708791,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005798392812721431
},
{
"step": 800,
"losses": {
"prediction_mse": 0.009408985264599323,
"latent_mse": 0.11286468803882599,
"scaled_prediction": 0.0013626806903630495,
"scaled_weighted_latent": 0.001634592073969543,
"total": 0.0029972726479172707
},
"gradients": {
"norms": {
"prediction": 0.002912086434662342,
"weighted_latent": 0.0029125665314495564,
"total": 5.200785381020978e-06
},
"prediction_weighted_latent_dot": -8.481631994072814e-06,
"prediction_weighted_latent_cosine": -0.9999984092458225,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0002596588456071913
},
{
"step": 900,
"losses": {
"prediction_mse": 0.0094091035425663,
"latent_mse": 0.11286351084709167,
"scaled_prediction": 0.001362697803415358,
"scaled_weighted_latent": 0.0016345750773325562,
"total": 0.0029972728807479143
},
"gradients": {
"norms": {
"prediction": 0.0029128547757864,
"weighted_latent": 0.002912570722401142,
"total": 3.2265631944028428e-06
},
"prediction_weighted_latent_dot": -8.483890269417316e-06,
"prediction_weighted_latent_cosine": -0.9999993789239304,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 9.733595106808934e-06
},
{
"step": 1000,
"losses": {
"prediction_mse": 0.009409232996404171,
"latent_mse": 0.1128622367978096,
"scaled_prediction": 0.0013627165462821722,
"scaled_weighted_latent": 0.0016345565672963858,
"total": 0.002997273113578558
},
"gradients": {
"norms": {
"prediction": 0.0029134999494999647,
"weighted_latent": 0.0029125595465302467,
"total": 9.305387720814906e-06
},
"prediction_weighted_latent_dot": -8.485700163873844e-06,
"prediction_weighted_latent_cosine": -0.9999950590228625,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00013846109504811466
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "84f3f61efe69cfb8f109725e0b62863a3c197f51886238d947df9d8dc3240346"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "0c60138bb638f11784fcfe838ffdaae05a2da62c6263090b727a58b9d26074ef"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "60cd5c13669bbe0de96ac584b56c31330afd0dd897e43e695677ad3a365f3c17"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "64399e82cd4edecbbd1334b2d2a2d885d878a6eaf5c9d9a973222def9cda2e11"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "a4aedf808d6ebcf878363e26ace828f34c789fe8720ed3da981ca429a440281c"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "05c847ae687a7b59790af4019f6e7795946c32dfe8ba4100826f763410faac6d"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "d2fa4d98d2d69f617fed60578d82d1b26a8e6acc0b8d29dbb1673f0312e11b4e"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "2a38473cfdf6a9bc698e342a3a977cbfd4fadca73de1505d3ed86fd67e75e761"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "9a26bfb6186e188a8ce57423da3ed695b7aaa73c8706db55d7771e6b90ad10ff"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "f72d265aec46e0b626d2109fbfac3943e73d2f4a26ed25030ca6fe4fb87ab0fc"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "ab7b2479826ff73a906f670b8db6e1130c3c6aa0816dbc06b152436568d135a0"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "fe943a3c3a39ef1a0d8246e36a9c3703472e67a373e38cf26ebb5945840de54e"
}
],
"final_losses": {
"prediction_mse": 0.009409232996404171,
"latent_mse": 0.1128622367978096,
"scaled_prediction": 0.0013627165462821722,
"scaled_weighted_latent": 0.0016345565672963858,
"total": 0.002997273113578558
},
"training_seconds": 3.096993900020607,
"evaluation_seconds": 0.49256959999911487
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 3,
"arm": "latent_only",
"status": "passed",
"files": {
"seed-3-latent_only/result.json": "ae9931d953b8170cc438315f9384d126a1a827943bf228553fc8772e9f115419",
"seed-3-latent_only/checkpoint.json": "47ee7af881cc62c1204047cb50ec63fbe8bd83e23095233d8a67330c15345ed2",
"inputs/seed-3-fourier/checkpoint.json": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"inputs/seed-3-fourier/result.json": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"seed-3-latent_only/diagnostics-arrays.npz": "aceda8f5bc2a00605c3e40b06fb7a9c70d8bce29d8f62ff2cee7356d0a5a4799"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "594eb3fc3f2d418f472a224e1ebd20a634875568e27274e27e4fcd1a62d6a88e",
"parent_checkpoint_sha256": "3f7a56dbb5ccecce92617cb879377d64244d1826ab372a930fea39da3cbb827c",
"parent_arrays_sha256": "f543cc91211283701db9cc9bcded1b27f05650b2352e5ec2f69758a2703a3101",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 5.811452865600586e-07,
"max_tolerance_ratio": 0.3990840958602454,
"worst_tolerance_index": [
60,
2
],
"actual_at_worst": 0.004562556743621826,
"saved_at_worst": 0.004561975598335266,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.00010696637036744505,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 0.00010696651588659734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014397905845926648
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001894764427561313,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 0.00018947666103485972,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-10,
"numeric_tolerance_ratio": 0.00021421976386698359
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2893109321594238,
"saved_at_worst": 1.2893109321594238,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4014429971381162,
"worst_tolerance_index": [
615,
4
],
"actual_at_worst": 0.0012289471924304962,
"saved_at_worst": 0.0012284964323043823,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 0.00010002103954320773,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"reported": {
"count": 784,
"observable_mse": 0.00010002109047491103,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.042732370857766e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 8.433461334789172e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005999956279993057
},
"reported": {
"count": 580,
"observable_mse": 8.433467883151025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0059999567456543446
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002910390913614281
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 0.00011633458780124784,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"reported": {
"count": 112,
"observable_mse": 0.00011633463145699352,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006062434520572424
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3153718491430616e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 0.00017905373533722013,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"reported": {
"count": 92,
"observable_mse": 0.00017905370623338968,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006442381534725428
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.8591882208963364e-05
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.000102343074104283,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"reported": {
"count": 98,
"observable_mse": 0.00010234319051960483,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.001757528884955003
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.67251874296926e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897133611142635
},
"reported": {
"count": 70,
"observable_mse": 8.67253402248025e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0058971368707716465
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.0020504503657808367
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010630184988258407,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897137802094221
},
"reported": {
"count": 14,
"observable_mse": 0.00010630159522406757,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897134076803923
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002343372258461114
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017647375352680683,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897135008126497
},
"reported": {
"count": 14,
"observable_mse": 0.00017647400090936571,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005897139199078083
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002636292941319088
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252822539769113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.737066673347726e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 70,
"observable_mse": 8.737093594390899e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968606099486351
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0005832209579303454
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010268489859299734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"reported": {
"count": 14,
"observable_mse": 0.0001026846730383113,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968607030808926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.255546860396862e-10,
"numeric_tolerance_ratio": 0.00022326212619665206
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017815936007536948,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"reported": {
"count": 14,
"observable_mse": 0.00017815965111367404,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005968604236841202
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.0002859439373556026
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001030568455462344,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580707758665
},
"reported": {
"count": 98,
"observable_mse": 0.00010305678733857349,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.00579952893372632
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.592774247517809e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058580242097378
},
"reported": {
"count": 70,
"observable_mse": 8.592770609539002e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585904866456985
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.006379481642109011
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001099406072171405,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585797764360905
},
"reported": {
"count": 14,
"observable_mse": 0.00010994031617883593,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060585867613554
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.004349647583201486
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00018181865743827075,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058583967387676
},
"reported": {
"count": 14,
"observable_mse": 0.00018181874474976212,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058590020984411
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.003769693806922108
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200838732998818,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097275298088789
},
"reported": {
"count": 98,
"observable_mse": 0.00010200848191743717,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006097279489040375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.002603515450311345
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543613512301818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060906685888767
},
"reported": {
"count": 72,
"observable_mse": 8.543617150280625e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006060910876840353
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026094108969952367
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0001222111313836649,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00609727343544364
},
"reported": {
"count": 14,
"observable_mse": 0.00012221158249303699,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060972776263952255
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0026035157515688335
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00017787207616493106,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311086472123861
},
"reported": {
"count": 12,
"observable_mse": 0.00017787201795727015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006311097647994757
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006851697621195759
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010200543329119682,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094702053815126
},
"reported": {
"count": 98,
"observable_mse": 0.00010200551332673058,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094708573073149
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004050559842515715
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.543937292415649e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058296654373407
},
"reported": {
"count": 72,
"observable_mse": 8.543945295969024e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006058302707970142
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.05359673500061e-09,
"numeric_tolerance_ratio": 0.0037697612537818566
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00012222511577419937,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060947067104279995
},
"reported": {
"count": 14,
"observable_mse": 0.00012222547957208008,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006094712298363447
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.0034719076328323475
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.0001778121804818511,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006308719515800476
},
"reported": {
"count": 12,
"observable_mse": 0.0001778119767550379,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0063087306916713715
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.006852692037580273
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067623384296894
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944067467004,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0023185078919724916
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 8.603723108535632e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067622918635607
},
"reported": {
"count": 70,
"observable_mse": 8.603732567280531e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067619659006596
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.259629011154175e-09,
"numeric_tolerance_ratio": 0.00202869440547593
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00010919466149061918,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067620124667883
},
"reported": {
"count": 14,
"observable_mse": 0.00010919448686763644,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.00028981346969819097
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00017808005213737488,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006067628972232342
},
"reported": {
"count": 14,
"observable_mse": 0.00017807971744332463,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00606762059032917
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.381903171539307e-09,
"numeric_tolerance_ratio": 0.005216642454567437
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875155178131536,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0121191280632047e-06
},
"reported": {
"count": 98,
"observable_mse": 0.00011875170457642525,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0175972420256585e-06
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.47811396245379e-09,
"numeric_tolerance_ratio": 0.005477556567808147
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.00010696427489165217,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0115189752468723e-06
},
"reported": {
"count": 84,
"observable_mse": 0.00010696444951463491,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0180974641116336e-06
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.5784888647613116e-09,
"numeric_tolerance_ratio": 0.006577819178658794
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.00018947529315482825,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0157128826904227e-06
},
"reported": {
"count": 14,
"observable_mse": 0.00018947523494716734,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.0145910209757858e-06
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.121861714636907e-09,
"numeric_tolerance_ratio": 0.0011217479031018784
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.69803557684645e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705172687768936
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.005974748847249882
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 4.724697646452114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754040114581585
},
"reported": {
"count": 72,
"observable_mse": 4.724687460111454e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008754054084420204
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-08,
"numeric_tolerance_ratio": 0.007448969996752607
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 6.864315218990669e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00870516151189804
},
"reported": {
"count": 14,
"observable_mse": 6.864342140033841e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008705178275704384
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6763806343078613e-08,
"numeric_tolerance_ratio": 0.008962120593553839
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 0.00018344067211728543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405925706028938
},
"reported": {
"count": 12,
"observable_mse": 0.00018344077398069203,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.008405914530158043
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1175870895385742e-08,
"numeric_tolerance_ratio": 0.0060718911179741195
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869952917099,
"saved_at_worst": 1.2869952917099,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.48761554050232203,
"worst_tolerance_index": [
3767,
3
],
"actual_at_worst": -0.00039064884185791016,
"saved_at_worst": -0.00039014220237731934,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.00011224200716242194,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010571317747235298
},
"reported": {
"count": 6272,
"observable_mse": 0.00011224204354221001,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010571316815912724
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004527287110249859
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 9.662971569923684e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01050430815666914
},
"reported": {
"count": 4092,
"observable_mse": 9.662973752710968e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010504309087991714
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00045420822063246434
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.00011953953799093142,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010571317747235298
},
"reported": {
"count": 896,
"observable_mse": 0.00011953959619859233,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010571317747235298
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.7520068339208964e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.0001569047017255798,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010782089084386826
},
"reported": {
"count": 1284,
"observable_mse": 0.00015690474538132548,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.010782088153064251
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00044813714952804585
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2941532135009766,
"saved_at_worst": 1.2941532135009766,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.47592563353345485,
"worst_tolerance_index": [
520,
2
],
"actual_at_worst": -0.0003317594528198242,
"saved_at_worst": -0.0003322511911392212,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00019525222887750715,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018011802807450294
},
"reported": {
"count": 3136,
"observable_mse": 0.00019525224342942238,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018011802807450294
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4273227263940613e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.0001820577308535576,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0181255042552948
},
"reported": {
"count": 1582,
"observable_mse": 0.00018205780361313373,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0181255042552948
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.145861626911538e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00019539719505701214,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018011802807450294
},
"reported": {
"count": 448,
"observable_mse": 0.00019539731147233397,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.018011802807450294
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011418419338689126
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.00021406661835499108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017847908660769463
},
"reported": {
"count": 1106,
"observable_mse": 0.00021406660380307585,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.017847908660769463
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4246935909883959e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2845898866653442,
"saved_at_worst": 1.2845898866653442,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.40870005807900295,
"worst_tolerance_index": [
1978,
3
],
"actual_at_worst": -0.00093841552734375,
"saved_at_worst": -0.0009379684925079346,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0004240799753461033,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028190135955810547
},
"reported": {
"count": 3136,
"observable_mse": 0.0004240800335537642,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028190135955810547
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.583961436031236e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.0004079900973010808,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02793256938457489
},
"reported": {
"count": 1001,
"observable_mse": 0.00040799021371640265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02793256938457489
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011185187479664834
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0004217948007863015,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028190141543745995
},
"reported": {
"count": 448,
"observable_mse": 0.0004217949463054538,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028190141543745995
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.000139629644445514
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.0004342339816503227,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028341861441731453
},
"reported": {
"count": 1687,
"observable_mse": 0.0004342339525464922,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.028341861441731453
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-11,
"numeric_tolerance_ratio": 2.7892637436628367e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.328358769416809,
"saved_at_worst": 1.328358769416809,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 6.407499313354492e-07,
"max_tolerance_ratio": 0.4102182073710469,
"worst_tolerance_index": [
788,
3
],
"actual_at_worst": 0.001624464988708496,
"saved_at_worst": 0.001623988151550293,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0010140761733055115,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04686106741428375
},
"reported": {
"count": 3136,
"observable_mse": 0.0010140760568901896,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04686106741428375
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00010569685666380313
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0009457325213588774,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04408523067831993
},
"reported": {
"count": 407,
"observable_mse": 0.0009457324049435556,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04408523067831993
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.0001063568133406557
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0010128346038982272,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04686106741428375
},
"reported": {
"count": 448,
"observable_mse": 0.0010128346038982272,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04686107113957405
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006551565462630186
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0010265145683661103,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04733925312757492
},
"reported": {
"count": 2281,
"observable_mse": 0.0010265145683661103,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04733925312757492
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011598924174904823
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010999653022736311,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010999666119460016,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00012954231484367922
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011603816412389278
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010919027408817783,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010919012856902555,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014394738889362153
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010782991535961628
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011893396003870293,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011893427290488034,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003091888621587732
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.010790933854877949
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011383872333681211,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011383901437511668,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.9103830456733704e-10,
"numeric_tolerance_ratio": 0.00028776244525315775
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.03806070610880852
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00042644605855457485,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.000426446640631184,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-10,
"numeric_tolerance_ratio": 0.0005582693981920547
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0369470976293087
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0004731984226964414,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00047319853911176324,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011115546162157263
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.035910893231630325
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0003548956592567265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00035489487345330417,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.8580342233181e-10,
"numeric_tolerance_ratio": 0.0007588714631438345
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020458154380321503
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00018904275202658027,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00018904254829976708,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.00019994696482166673
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 2.021566388066276e-06
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011874968186020851,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011874980555148795,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2369127944111824e-10,
"numeric_tolerance_ratio": 0.00012223968555211934
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006445486564189196
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00012876497930847108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00012876487744506449,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.00010056843833486492
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011875350901391357,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011875366908498108,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6007106751203537e-10,
"numeric_tolerance_ratio": 0.0001581924738429869
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.008087845519185066
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 9.686173143563792e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 9.68617750913836e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.3655745685100555e-11,
"numeric_tolerance_ratio": 4.3236944961253016e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.698037759633735e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.698032666463405e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.059283086545828e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012188127264380455
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00014413957251235843,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00014413951430469751,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.738058002000685e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010252827632939443,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010252842912450433,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.5279510989785194e-10,
"numeric_tolerance_ratio": 0.00015124442457133805
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00011474297207314521,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00011474302300484851,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.035392711752096e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010750066576292738,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00010750071669463068,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.0390007111411554e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00010249445040244609,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001024944576784037,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2021396741805025e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.00014975426893215626,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0001497541816206649,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.602325712311944e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 0.00010700838174670935,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 0.00010700869461288676,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.128661774098873e-10,
"numeric_tolerance_ratio": 0.0003095536838477713
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 6.998753087827936e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 6.998747994657606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.057772256490849e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.253204239532351e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.253222429426387e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-10,
"numeric_tolerance_ratio": 0.0001805890875343717
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0002318347105756402,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.00023183472512755543,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.422219535332207e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.011943741701543331,
"latent_mse": 0.212107315659523,
"scaled_prediction": 0.0017297833692282438,
"scaled_weighted_latent": 0.0030718990601599216,
"total": 0.0030718990601599216
},
"gradients": {
"norms": {
"prediction": 0.0034363465383648872,
"weighted_latent": 0.004243341740220785,
"total": 0.004243341740220785
},
"prediction_weighted_latent_dot": -1.2367510862532072e-05,
"prediction_weighted_latent_cosine": -0.8481591220444237,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.012406026013195515,
"latent_mse": 0.20755654573440552,
"scaled_prediction": 0.0017967348685488105,
"scaled_weighted_latent": 0.003005991457030177,
"total": 0.003005991457030177
},
"gradients": {
"norms": {
"prediction": 0.004003142472356558,
"weighted_latent": 0.0041850716806948185,
"total": 0.0041850716806948185
},
"prediction_weighted_latent_dot": -1.3552008567785379e-05,
"prediction_weighted_latent_cosine": -0.8089090973549798,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.031445253640413284
},
{
"step": 100,
"losses": {
"prediction_mse": 0.027742937207221985,
"latent_mse": 0.04495343565940857,
"scaled_prediction": 0.00401794258505106,
"scaled_weighted_latent": 0.0006510497769340873,
"total": 0.0006510497769340873
},
"gradients": {
"norms": {
"prediction": 0.011764733120799065,
"weighted_latent": 0.0018284975085407495,
"total": 0.0018284975085407495
},
"prediction_weighted_latent_dot": 1.0727669177867938e-05,
"prediction_weighted_latent_cosine": 0.4986880018611385,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.005691013298928738
},
{
"step": 200,
"losses": {
"prediction_mse": 0.007840918377041817,
"latent_mse": 0.008958370424807072,
"scaled_prediction": 0.0011355812894180417,
"scaled_weighted_latent": 0.0001297419221373275,
"total": 0.0001297419221373275
},
"gradients": {
"norms": {
"prediction": 0.008955360390245914,
"weighted_latent": 0.000795930449385196,
"total": 0.000795930449385196
},
"prediction_weighted_latent_dot": 4.943171006743796e-06,
"prediction_weighted_latent_cosine": 0.6935015683567417,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0028745271265506744
},
{
"step": 300,
"losses": {
"prediction_mse": 0.001478584366850555,
"latent_mse": 0.0014538821997120976,
"scaled_prediction": 0.00021413981448858976,
"scaled_weighted_latent": 2.105622479575686e-05,
"total": 2.105622479575686e-05
},
"gradients": {
"norms": {
"prediction": 0.00454098591580987,
"weighted_latent": 0.0003189259150531143,
"total": 0.0003189259150531143
},
"prediction_weighted_latent_dot": 9.54590291257773e-07,
"prediction_weighted_latent_cosine": 0.659139059299341,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0013068236876279116
},
{
"step": 400,
"losses": {
"prediction_mse": 0.0002792561717797071,
"latent_mse": 0.0001837616873672232,
"scaled_prediction": 4.044399975100532e-05,
"scaled_weighted_latent": 2.6613761292537674e-06,
"total": 2.6613761292537674e-06
},
"gradients": {
"norms": {
"prediction": 0.0016344879986718297,
"weighted_latent": 0.00011334959708619863,
"total": 0.00011334959708619863
},
"prediction_weighted_latent_dot": 1.2311144814702857e-07,
"prediction_weighted_latent_cosine": 0.6645026589731049,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005251585971564054
},
{
"step": 500,
"losses": {
"prediction_mse": 0.0001206986780744046,
"latent_mse": 1.7532936908537522e-05,
"scaled_prediction": 1.7480499082012102e-05,
"scaled_weighted_latent": 2.5392529323653434e-07,
"total": 2.5392529323653434e-07
},
"gradients": {
"norms": {
"prediction": 0.0005504697328433394,
"weighted_latent": 3.501208630041219e-05,
"total": 3.501208630041219e-05
},
"prediction_weighted_latent_dot": 1.0782380321927576e-08,
"prediction_weighted_latent_cosine": 0.5594524905161186,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0001831014669733122
},
{
"step": 600,
"losses": {
"prediction_mse": 0.0001069319769158028,
"latent_mse": 1.23396443996171e-06,
"scaled_prediction": 1.548670115880668e-05,
"scaled_weighted_latent": 1.787121028939964e-08,
"total": 1.787121028939964e-08
},
"gradients": {
"norms": {
"prediction": 0.0003244292747694999,
"weighted_latent": 9.28842473513214e-06,
"total": 9.28842473513214e-06
},
"prediction_weighted_latent_dot": 4.0361963771218257e-10,
"prediction_weighted_latent_cosine": 0.13393996656636023,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.463738852995448e-05
},
{
"step": 700,
"losses": {
"prediction_mse": 0.0001066865079337731,
"latent_mse": 6.252758311120488e-08,
"scaled_prediction": 1.5451149010914378e-05,
"scaled_weighted_latent": 9.055718952488689e-10,
"total": 9.055718952488689e-10
},
"gradients": {
"norms": {
"prediction": 0.00031490897526964545,
"weighted_latent": 2.0908673832309432e-06,
"total": 2.0908673832309432e-06
},
"prediction_weighted_latent_dot": -7.719813677198317e-11,
"prediction_weighted_latent_cosine": -0.11724525942827876,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.3811811186315026e-05
},
{
"step": 800,
"losses": {
"prediction_mse": 0.00010689532064134255,
"latent_mse": 2.2136834587627163e-09,
"scaled_prediction": 1.548139152873773e-05,
"scaled_weighted_latent": 3.206024334900803e-11,
"total": 3.206024334900803e-11
},
"gradients": {
"norms": {
"prediction": 0.00031767296604812145,
"weighted_latent": 3.934148367079615e-07,
"total": 3.934148367079615e-07
},
"prediction_weighted_latent_dot": -2.4161134898337266e-11,
"prediction_weighted_latent_cosine": -0.19332425172822873,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.892859583880636e-06
},
{
"step": 900,
"losses": {
"prediction_mse": 0.00010695397213567048,
"latent_mse": 5.3057332832784354e-11,
"scaled_prediction": 1.548988620925229e-05,
"scaled_weighted_latent": 7.684165803656384e-13,
"total": 7.684165803656384e-13
},
"gradients": {
"norms": {
"prediction": 0.00031847888021729887,
"weighted_latent": 6.091948279163262e-08,
"total": 6.091948279163262e-08
},
"prediction_weighted_latent_dot": -4.365838853631621e-12,
"prediction_weighted_latent_cosine": -0.22502504459760064,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 4.838051381739206e-07
},
{
"step": 1000,
"losses": {
"prediction_mse": 0.00010696444951463491,
"latent_mse": 1.0365225388067612e-12,
"scaled_prediction": 1.5491403246414848e-05,
"scaled_weighted_latent": 1.5011705732106105e-14,
"total": 1.5011705732106105e-14
},
"gradients": {
"norms": {
"prediction": 0.00031862015021033585,
"weighted_latent": 8.613119106826161e-09,
"total": 8.613119106826161e-09
},
"prediction_weighted_latent_dot": -7.386099110801014e-13,
"prediction_weighted_latent_cosine": -0.2691419781087464,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.789324812762061e-07
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "2609ad2704a6063d91847fcb51c52e57f5c8517e276eeac245d2e327edc58ff3"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "0e3d2dd4cd57267cdee0643d8eb23e0eedae1c115d6523a89036f6aac1a3c9f7"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "190491187f8fa1bc836b3998b5b65b4c4d8d66ee4002184fe47efdee70e0673d"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "87b1abc43b8d15560398941b9ef832fad3edc7904f08de0aedeb6e5c40d30b58"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "c6895352e70635b188d04d580c8d801cc9a80f9f364141413686848987776b5e"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "8489835f222439a3081472475cd77a99349a411fa927df6e32da8ef302dbaf22"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "3cdec08a90488cf6c84fd797c4c3426ac47fc64150ef8179e962c6f21edcf853"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "91250f181ff3f14daf95381550c85612b8d4984e7ff0d71f69da0890e3584655"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "2681ea448321713221cb5be69c7df1d8569752ebbbd5b9e197f3408316bd75cc"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "515e6f3c6c67a5a1e2deec322265e5843cf03a3f6982bd682f6af5a09a1801fd"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "ccc418c8dbd904af9ff6babf08f282bb312f490a7bb966d09f2aef1bbc724571"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "47ee7af881cc62c1204047cb50ec63fbe8bd83e23095233d8a67330c15345ed2"
}
],
"final_losses": {
"prediction_mse": 0.00010696444951463491,
"latent_mse": 1.0365225388067612e-12,
"scaled_prediction": 1.5491403246414848e-05,
"scaled_weighted_latent": 1.5011705732106105e-14,
"total": 1.5011705732106105e-14
},
"training_seconds": 2.7897566000465304,
"evaluation_seconds": 0.4940434000454843
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 4,
"arm": "native",
"status": "passed",
"files": {
"seed-4-native/result.json": "9bbda5214373b1ace29e9b95c30b3eb6e115f9f4fd512100b05bb6468ef36a92",
"seed-4-native/checkpoint.json": "2b443c6141bcba042a8d14a0d7e0d065b831ee6ad11c793c91e2bf196bf229ad",
"inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"seed-4-native/diagnostics-arrays.npz": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.0978193283081055e-07,
"max_tolerance_ratio": 0.3144731636939299,
"worst_tolerance_index": [
0,
8
],
"actual_at_worst": 0.00018735229969024658,
"saved_at_worst": 0.0001876726746559143,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450094886356965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.3450114895240404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.996207215045689e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800791404908523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 5.800786675536074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.702096589973028e-05
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2923264503479004,
"saved_at_worst": 1.2923264503479004,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.37291832992681323,
"worst_tolerance_index": [
343,
9
],
"actual_at_worst": 0.001188725233078003,
"saved_at_worst": 0.0011883080005645752,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 8.654144039610401e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043472398072481155
},
"reported": {
"count": 784,
"observable_mse": 8.654144039610401e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043472398072481155
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 9.035727271111682e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04677638038992882
},
"reported": {
"count": 580,
"observable_mse": 9.035727271111682e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.046776384115219116
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006561337704954932
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 9.340440010419115e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04347239434719086
},
"reported": {
"count": 112,
"observable_mse": 9.340439282823354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043472398072481155
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006966753750995664
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 5.4130203352542594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003164678579196334
},
"reported": {
"count": 92,
"observable_mse": 5.413021062850021e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031646783463656902
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017686010818346055
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5274550353060476e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211733050644398
},
"reported": {
"count": 98,
"observable_mse": 2.5274570361943915e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171675249934
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012327307636645429
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0186165784252807e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"reported": {
"count": 70,
"observable_mse": 2.0186189431115054e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211709767580032
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002817670465808772
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1840363842784427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171209588647
},
"reported": {
"count": 14,
"observable_mse": 2.1840449335286394e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221172373741865
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0008805219275269677
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.415066334535368e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221169812604785
},
"reported": {
"count": 14,
"observable_mse": 5.415061968960799e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.003698191444345532
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493371140328236e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220667831599712
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.008045871341972934
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9936096578021534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022065619006753
},
"reported": {
"count": 70,
"observable_mse": 1.9936062017222866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220558401197195
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.778887033462524e-09,
"numeric_tolerance_ratio": 0.007509480187709456
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.3793707441654988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022068180143833
},
"reported": {
"count": 14,
"observable_mse": 2.3793658328941092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009118654187569325
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.497939855558798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022070275619626
},
"reported": {
"count": 14,
"observable_mse": 5.497950769495219e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220600310713053
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.007867071950464247
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5550732971169055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836770940572023
},
"reported": {
"count": 98,
"observable_mse": 2.55507566180313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836845446377993
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.005651364435865696
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.939376124937553e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836775597184896
},
"reported": {
"count": 70,
"observable_mse": 1.9393763068364933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836838461458683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004768338995395631
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.679098724911455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836782582104206
},
"reported": {
"count": 14,
"observable_mse": 2.679110002645757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836875714361668
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.007064203922984993
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.509533730219118e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836743000894785
},
"reported": {
"count": 14,
"observable_mse": 5.509537731995806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003183686174452305
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009006860956200402
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5143033781205304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363468684256077
},
"reported": {
"count": 98,
"observable_mse": 2.514301559131127e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634756691753864
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.005226873219612861
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949908801179845e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588928636163473
},
"reported": {
"count": 72,
"observable_mse": 1.949909346876666e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588993828743696
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004880086177358148
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.875245627365075e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634670544415712
},
"reported": {
"count": 14,
"observable_mse": 2.8752372600138187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363472642377019
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004181499522790897
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.4795717005617917e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033907967153936625
},
"reported": {
"count": 12,
"observable_mse": 5.4795644246041775e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003390809055417776
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.00921529390986458
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5130846552201547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 98,
"observable_mse": 2.5130850190180354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750871662050486
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0031334013256818006
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949693978531286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003370406571775675
},
"reported": {
"count": 72,
"observable_mse": 1.949694888025988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033704107627272606
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.003134497256772981
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750897273421288
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.752088665962219e-09,
"numeric_tolerance_ratio": 0.005048256724707581
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.473628334584646e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003403007285669446
},
"reported": {
"count": 12,
"observable_mse": 5.4736312449676916e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034030096139758825
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0017371519558643553
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509041734039783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103164955973625
},
"reported": {
"count": 98,
"observable_mse": 2.5509052647976205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103166352957487
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0010661422012763408
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9929024347220547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031647231429815
},
"reported": {
"count": 70,
"observable_mse": 1.9929013433284126e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031661201268435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.001066142220220661
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.406693783996161e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103160299360752
},
"reported": {
"count": 14,
"observable_mse": 2.406714338576421e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031658872962
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004264568956659927
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.485124347615056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103169146925211
},
"reported": {
"count": 14,
"observable_mse": 5.485117799253203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031689140945673
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001776903321481438
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0005257376469671726,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265803664922714
},
"reported": {
"count": 98,
"observable_mse": 0.0005257370648905635,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265802174806595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0011232763007838916
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 0.0005169388605281711,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265803664922714
},
"reported": {
"count": 84,
"observable_mse": 0.0005169383366592228,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265802919864655
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0005616381188482063
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 0.0005785307730548084,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265803664922714
},
"reported": {
"count": 14,
"observable_mse": 0.0005785297835245728,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.12265802174806595
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0011232763007838916
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492266927845776e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384734462946653
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006583761406554621
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.54845256981207e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003532622940838337
},
"reported": {
"count": 72,
"observable_mse": 8.548483492631931e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035326124634593725
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.0077423180429610685
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4730835573573131e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003438472282141447
},
"reported": {
"count": 14,
"observable_mse": 1.4730812836205587e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.683411240577698e-09,
"numeric_tolerance_ratio": 0.005717477010955329
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.987682041246444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080525808036327
},
"reported": {
"count": 12,
"observable_mse": 4.987671127310023e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080549091100693
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0018178454519917982
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2935075759887695,
"saved_at_worst": 1.2935075759887695,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.5794924900882756,
"worst_tolerance_index": [
5875,
12
],
"actual_at_worst": 0.0005421936511993408,
"saved_at_worst": 0.0005428045988082886,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 0.0003078756562899798,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0645698681473732
},
"reported": {
"count": 6272,
"observable_mse": 0.00030787562718614936,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0645698606967926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0009991410104973272
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 0.00035424259840510786,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.06916395574808121
},
"reported": {
"count": 4092,
"observable_mse": 0.00035424259840510786,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0691639631986618
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0009411581098114823
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 0.00031304298317991197,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0645698606967926
},
"reported": {
"count": 896,
"observable_mse": 0.00031304298317991197,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0645698606967926
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 0.00015650216664653271,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04702414572238922
},
"reported": {
"count": 1284,
"observable_mse": 0.0001565021084388718,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.04702414199709892
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006532830075113513
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.1930153369903564,
"saved_at_worst": 1.1930153369903564,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4165565519257677,
"worst_tolerance_index": [
1728,
8
],
"actual_at_worst": 1.5810132026672363e-05,
"saved_at_worst": 1.6227364540100098e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0007142377435229719,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08957353979349136
},
"reported": {
"count": 3136,
"observable_mse": 0.00071423762710765,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08957353234291077
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.0007482491000987653
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 0.0007724000606685877,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09334152936935425
},
"reported": {
"count": 1582,
"observable_mse": 0.0007724001188762486,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.09334152936935425
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.4034068797232435e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0007104201940819621,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08957353979349136
},
"reported": {
"count": 448,
"observable_mse": 0.0007104201940819621,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08957353979349136
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 0.0006325896247290075,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08389020711183548
},
"reported": {
"count": 1106,
"observable_mse": 0.0006325897411443293,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.08389020711183548
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.0001094891504902602
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.3042800426483154,
"saved_at_worst": 1.3042800426483154,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.409652597231079,
"worst_tolerance_index": [
1727,
0
],
"actual_at_worst": -0.0006401576101779938,
"saved_at_worst": -0.0006397217512130737,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.005111921578645706,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.18345704674720764
},
"reported": {
"count": 3136,
"observable_mse": 0.005111921112984419,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.18345703184604645
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0007702568912411534
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00642401585355401,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.2068207710981369
},
"reported": {
"count": 1001,
"observable_mse": 0.00642401585355401,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.2068207710981369
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.005084032658487558,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.18345704674720764
},
"reported": {
"count": 448,
"observable_mse": 0.005084032658487558,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.18345703184604645
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4901161193847656e-08,
"numeric_tolerance_ratio": 0.0007702568912411534
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.004340781830251217,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16806550323963165
},
"reported": {
"count": 1687,
"observable_mse": 0.004340781830251217,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.16806550323963165
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2052175998687744,
"saved_at_worst": 1.2052175998687744,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.662441253662109e-07,
"max_tolerance_ratio": 0.42896605801972054,
"worst_tolerance_index": [
3103,
0
],
"actual_at_worst": -0.0004641897976398468,
"saved_at_worst": -0.00046463869512081146,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.014956056140363216,
"exact_state_accuracy": 0.9764030612244898,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9920280612244898,
"h": 0.9808673469387755
},
"normalized_latent_rmse": 0.2683657109737396
},
"reported": {
"count": 3136,
"observable_mse": 0.014956055209040642,
"exact_state_accuracy": 0.9764030575752258,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9920280575752258,
"h": 0.980867326259613
},
"normalized_latent_rmse": 0.2683657109737396
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.0679162493486558e-08,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00037318501133868916
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.020802950486540794,
"exact_state_accuracy": 0.9533169533169533,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9778869778869779,
"h": 0.9680589680589681
},
"normalized_latent_rmse": 0.3151726722717285
},
"reported": {
"count": 407,
"observable_mse": 0.020802950486540794,
"exact_state_accuracy": 0.9533169269561768,
"per_field_accuracy": {
"x": 1.0,
"y": 0.977886974811554,
"h": 0.9680589437484741
},
"normalized_latent_rmse": 0.3151726722717285
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.6360776561595856e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.015006010420620441,
"exact_state_accuracy": 0.9776785714285714,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9933035714285714,
"h": 0.9821428571428571
},
"normalized_latent_rmse": 0.2683657109737396
},
"reported": {
"count": 448,
"observable_mse": 0.015006013214588165,
"exact_state_accuracy": 0.9776785969734192,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9933035969734192,
"h": 0.9821429252624512
},
"normalized_latent_rmse": 0.2683657109737396
},
"differences": [],
"max_errors": {
"accuracy_abs": 6.811959407659884e-08,
"numeric_abs": 2.7939677238464355e-09,
"numeric_tolerance_ratio": 0.0011173183425402945
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.013902980834245682,
"exact_state_accuracy": 0.9802718106093818,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9943007452871547,
"h": 0.9829022358614643
},
"normalized_latent_rmse": 0.25912654399871826
},
"reported": {
"count": 2281,
"observable_mse": 0.013902980834245682,
"exact_state_accuracy": 0.9802718162536621,
"per_field_accuracy": {
"x": 1.0,
"y": 0.9943007230758667,
"h": 0.9829022288322449
},
"normalized_latent_rmse": 0.25912654399871826
},
"differences": [],
"max_errors": {
"accuracy_abs": 2.2211288031037668e-08,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006257921922951937
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5511306375847198e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5511330022709444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3586689464189724e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006255257874727249
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5392424504389055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5392444513272494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9958204678516046e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006481279619038105
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5999313947977498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5999332137871534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8142724164315183e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006476297043263912
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.6249348593410105e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.624933586048428e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2699590243860832e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.021607279777526855
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.766223668819293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.7661858338397e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.7834979593753815e-10,
"numeric_tolerance_ratio": 0.00037580701583076647
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.019717823714017868
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.143248901935294e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.143239443190396e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.40099235142008e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020352382212877274
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.072931475704536e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.072944572428241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00013004741886984787
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012590174563229084
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.74880874005612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.748832386918366e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-10,
"numeric_tolerance_ratio": 0.000235585452088286
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.2127009630203247
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.006515105254948139,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.006515105254948139,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.21456055343151093
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.008191288448870182,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.008191288448870182,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0037216152995824814
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5795047147548757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.579504350956995e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6322416655772068e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492263289866969e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.185452315956354e-12,
"numeric_tolerance_ratio": 8.17360690293936e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006914579775184393
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1585754186380655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.158584877382964e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.428962719383602e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493387511232868e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.447291564678985e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5946865207515657e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.594685065560043e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4514255306884354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5181043383781798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5181052478728816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.072102508792652e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0015580033650621772,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0015580025501549244,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.149072527885437e-10,
"numeric_tolerance_ratio": 0.0007050588968572434
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 0.0005676934379152954,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 0.0005676930886693299,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.4924596548080444e-10,
"numeric_tolerance_ratio": 0.00033048458405294303
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.8929631298524328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.8929516702191904e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1459633242338896e-10,
"numeric_tolerance_ratio": 0.00011426576708166119
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.3614365343528334e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.3614378076454159e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2715614306756342e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4222289792087395e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4222278878150973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0898436361148734e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.5869325074600056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.586930688470602e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8124881342346815e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 91,
"error_count": 7,
"per_field_correct_counts": {
"x": 98,
"y": 91,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 78,
"error_count": 20,
"per_field_correct_counts": {
"x": 98,
"y": 87,
"h": 85
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 51,
"error_count": 47,
"per_field_correct_counts": {
"x": 98,
"y": 91,
"h": 51
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3062,
"accuracy": 0.9764030612244898
},
"numpy": {
"cases": 3136,
"correct": 3062,
"accuracy": 0.9764030612244898
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0005169383366592228,
"latent_mse": 0.015044992789626122,
"scaled_prediction": 7.486693357350305e-05,
"scaled_weighted_latent": 0.00021789300080854446,
"total": 0.0002927599416580051
},
"gradients": {
"norms": {
"prediction": 0.001070561120286584,
"weighted_latent": 0.0012635241728276014,
"total": 0.002058987971395254
},
"prediction_weighted_latent_dot": 7.48418131024664e-07,
"prediction_weighted_latent_cosine": 0.5532854864566016,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "2b443c6141bcba042a8d14a0d7e0d065b831ee6ad11c793c91e2bf196bf229ad"
}
],
"final_losses": {
"prediction_mse": 0.0005169383366592228,
"latent_mse": 0.015044992789626122,
"scaled_prediction": 7.486693357350305e-05,
"scaled_weighted_latent": 0.00021789300080854446,
"total": 0.0002927599416580051
},
"training_seconds": 4.1199964471161366e-05,
"evaluation_seconds": 0.5248835000093095
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 4,
"arm": "prediction_latent",
"status": "passed",
"files": {
"seed-4-prediction_latent/result.json": "0afedbfab8403dc148a8ab961ba49342d0bdd11d2011d96f92da68f55931d6b8",
"seed-4-prediction_latent/checkpoint.json": "a0d56a6c0cb366f7ef9baa918d85d54904469a947720c2e1064b3d1c99352424",
"inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"seed-4-prediction_latent/diagnostics-arrays.npz": "ab86bd115de46443231d8b433015c3dd00a97ac893cfcfcc5e8dede70da8b70d"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.0978193283081055e-07,
"max_tolerance_ratio": 0.3144731636939299,
"worst_tolerance_index": [
0,
8
],
"actual_at_worst": 0.00018735229969024658,
"saved_at_worst": 0.0001876726746559143,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450094886356965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.3450114895240404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.996207215045689e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800791404908523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 5.800786675536074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.702096589973028e-05
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2923264503479004,
"saved_at_worst": 1.2923264503479004,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.37291832992681323,
"worst_tolerance_index": [
343,
9
],
"actual_at_worst": 0.001188725233078003,
"saved_at_worst": 0.0011883080005645752,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.3999598852242343e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003208009758964181
},
"reported": {
"count": 784,
"observable_mse": 2.3999595214263536e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003208009758964181
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6292687090974994e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.8472606825525872e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00321482983417809
},
"reported": {
"count": 580,
"observable_mse": 1.8472603187547065e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00321482983417809
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.631270904443287e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.7871383281308226e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032080088276416063
},
"reported": {
"count": 112,
"observable_mse": 2.7871372367371805e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032080088276416063
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.088360232795705e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 5.4130203352542594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003164678579196334
},
"reported": {
"count": 92,
"observable_mse": 5.413021062850021e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031646783463656902
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017686010818346055
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5274550353060476e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211733050644398
},
"reported": {
"count": 98,
"observable_mse": 2.5274570361943915e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171675249934
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012327307636645429
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0186165784252807e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"reported": {
"count": 70,
"observable_mse": 2.0186189431115054e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211709767580032
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002817670465808772
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1840363842784427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171209588647
},
"reported": {
"count": 14,
"observable_mse": 2.1840449335286394e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221172373741865
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0008805219275269677
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.415066334535368e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221169812604785
},
"reported": {
"count": 14,
"observable_mse": 5.415061968960799e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.003698191444345532
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493371140328236e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220667831599712
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.008045871341972934
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9936096578021534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022065619006753
},
"reported": {
"count": 70,
"observable_mse": 1.9936062017222866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220558401197195
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.778887033462524e-09,
"numeric_tolerance_ratio": 0.007509480187709456
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.3793707441654988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022068180143833
},
"reported": {
"count": 14,
"observable_mse": 2.3793658328941092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009118654187569325
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.497939855558798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022070275619626
},
"reported": {
"count": 14,
"observable_mse": 5.497950769495219e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220600310713053
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.007867071950464247
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5550732971169055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836770940572023
},
"reported": {
"count": 98,
"observable_mse": 2.55507566180313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836845446377993
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.005651364435865696
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.939376124937553e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836775597184896
},
"reported": {
"count": 70,
"observable_mse": 1.9393763068364933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836838461458683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004768338995395631
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.679098724911455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836782582104206
},
"reported": {
"count": 14,
"observable_mse": 2.679110002645757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836875714361668
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.007064203922984993
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.509533730219118e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836743000894785
},
"reported": {
"count": 14,
"observable_mse": 5.509537731995806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003183686174452305
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009006860956200402
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5143033781205304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363468684256077
},
"reported": {
"count": 98,
"observable_mse": 2.514301559131127e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634756691753864
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.005226873219612861
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949908801179845e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588928636163473
},
"reported": {
"count": 72,
"observable_mse": 1.949909346876666e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588993828743696
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004880086177358148
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.875245627365075e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634670544415712
},
"reported": {
"count": 14,
"observable_mse": 2.8752372600138187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363472642377019
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004181499522790897
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.4795717005617917e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033907967153936625
},
"reported": {
"count": 12,
"observable_mse": 5.4795644246041775e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003390809055417776
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.00921529390986458
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5130846552201547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 98,
"observable_mse": 2.5130850190180354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750871662050486
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0031334013256818006
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949693978531286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003370406571775675
},
"reported": {
"count": 72,
"observable_mse": 1.949694888025988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033704107627272606
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.003134497256772981
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750897273421288
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.752088665962219e-09,
"numeric_tolerance_ratio": 0.005048256724707581
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.473628334584646e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003403007285669446
},
"reported": {
"count": 12,
"observable_mse": 5.4736312449676916e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034030096139758825
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0017371519558643553
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509041734039783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103164955973625
},
"reported": {
"count": 98,
"observable_mse": 2.5509052647976205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103166352957487
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0010661422012763408
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9929024347220547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031647231429815
},
"reported": {
"count": 70,
"observable_mse": 1.9929013433284126e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031661201268435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.001066142220220661
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.406693783996161e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103160299360752
},
"reported": {
"count": 14,
"observable_mse": 2.406714338576421e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031658872962
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004264568956659927
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.485124347615056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103169146925211
},
"reported": {
"count": 14,
"observable_mse": 5.485117799253203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031689140945673
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001776903321481438
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5402956453035586e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0029204681050032377
},
"reported": {
"count": 98,
"observable_mse": 2.5402905521332286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0029204622842371464
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-09,
"numeric_tolerance_ratio": 0.004505075718883546
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.0592340661096387e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0029204683378338814
},
"reported": {
"count": 84,
"observable_mse": 2.059229154838249e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0029204629827290773
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.3551048040390015e-09,
"numeric_tolerance_ratio": 0.004144669437308267
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.426665302366018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.002920466475188732
},
"reported": {
"count": 14,
"observable_mse": 5.4266613005893305e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0029204594902694225
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.005406092031693245
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492266927845776e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384734462946653
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006583761406554621
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.54845256981207e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003532622940838337
},
"reported": {
"count": 72,
"observable_mse": 8.548483492631931e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035326124634593725
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.0077423180429610685
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4730835573573131e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003438472282141447
},
"reported": {
"count": 14,
"observable_mse": 1.4730812836205587e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.683411240577698e-09,
"numeric_tolerance_ratio": 0.005717477010955329
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.987682041246444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080525808036327
},
"reported": {
"count": 12,
"observable_mse": 4.987671127310023e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080549091100693
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0018178454519917982
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2935075759887695,
"saved_at_worst": 1.2935075759887695,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.5794924900882756,
"worst_tolerance_index": [
5875,
12
],
"actual_at_worst": 0.0005421936511993408,
"saved_at_worst": 0.0005428045988082886,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.4210245101130567e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006091923452913761
},
"reported": {
"count": 6272,
"observable_mse": 2.421025237708818e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006091923452913761
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2583848812010335e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.883342702058144e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006118892692029476
},
"reported": {
"count": 4092,
"observable_mse": 1.8833430658560246e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.006118892692029476
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6311401245170528e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.712010063987691e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060919225215911865
},
"reported": {
"count": 896,
"observable_mse": 2.7120109734823927e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060919225215911865
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.070348134056416e-06
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.9315156755037606e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0060051665641367435
},
"reported": {
"count": 1284,
"observable_mse": 3.9315171306952834e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.006005167029798031
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002909443471853698
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869195938110352,
"saved_at_worst": 1.2869195938110352,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4165565519257677,
"worst_tolerance_index": [
1728,
8
],
"actual_at_worst": 1.5810132026672363e-05,
"saved_at_worst": 1.6227364540100098e-05,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.7332403735490516e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011979796923696995
},
"reported": {
"count": 3136,
"observable_mse": 3.7332429201342165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011979796923696995
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5371135039380457e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 3.2660438591847196e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012036066502332687
},
"reported": {
"count": 1582,
"observable_mse": 3.266046041972004e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.012036066502332687
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.1756814086018716e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.7948124372633174e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011979799717664719
},
"reported": {
"count": 448,
"observable_mse": 3.7948160752421245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011979798786342144
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.00042371751610127835
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 4.376568176667206e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011898848228156567
},
"reported": {
"count": 1106,
"observable_mse": 4.37657035945449e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.011898848228156567
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.1827872842550278e-11,
"numeric_tolerance_ratio": 2.173275789850249e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2849032878875732,
"saved_at_worst": 1.2849032878875732,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4184826686608572,
"worst_tolerance_index": [
1883,
7
],
"actual_at_worst": -0.0005046799778938293,
"saved_at_worst": -0.0005042403936386108,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0001141131142503582,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02400389313697815
},
"reported": {
"count": 3136,
"observable_mse": 0.00011411319428589195,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02400389313697815
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.913252721082345e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 0.00010936548642348498,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02393919788300991
},
"reported": {
"count": 1001,
"observable_mse": 0.00010936558828689158,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02393919788300991
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0186340659856796e-10,
"numeric_tolerance_ratio": 0.00010076142336428203
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0001147212678915821,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02400389313697815
},
"reported": {
"count": 448,
"observable_mse": 0.00011472133337520063,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02400389313697815
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.548361852765083e-11,
"numeric_tolerance_ratio": 6.474090226448136e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 0.00011676864960463718,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.024042200297117233
},
"reported": {
"count": 1687,
"observable_mse": 0.00011676873691612855,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.024042200297117233
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.731149137020111e-11,
"numeric_tolerance_ratio": 8.630373357414125e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2667545080184937,
"saved_at_worst": 1.2667545080184937,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.42896605801972054,
"worst_tolerance_index": [
3103,
0
],
"actual_at_worst": -0.0004641897976398468,
"saved_at_worst": -0.00046463869512081146,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.00047022575745359063,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04926556348800659
},
"reported": {
"count": 3136,
"observable_mse": 0.00047022593207657337,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04926556348800659
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016678052973568357
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0004428043612278998,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.048294827342033386
},
"reported": {
"count": 407,
"observable_mse": 0.0004428044776432216,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.048294827342033386
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011147898256274539
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.00047217815881595016,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04926556348800659
},
"reported": {
"count": 448,
"observable_mse": 0.0004721784498542547,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04926555976271629
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006285759070490511
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.0004747350758407265,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.049436770379543304
},
"reported": {
"count": 2281,
"observable_mse": 0.0004747352795675397,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.049436770379543304
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.0372681319713593e-10,
"numeric_tolerance_ratio": 0.00019449351965441462
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006257921922951937
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5511306375847198e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5511330022709444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3586689464189724e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006255257874727249
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5392424504389055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5392444513272494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9958204678516046e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006481279619038105
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5999313947977498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5999332137871534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8142724164315183e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006476297043263912
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.6249348593410105e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.624933586048428e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2699590243860832e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.021607279777526855
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.766223668819293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.7661858338397e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.7834979593753815e-10,
"numeric_tolerance_ratio": 0.00037580701583076647
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.019717823714017868
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.143248901935294e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.143239443190396e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.40099235142008e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020352382212877274
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.072931475704536e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.072944572428241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00013004741886984787
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012590174563229084
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.74880874005612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.748832386918366e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-10,
"numeric_tolerance_ratio": 0.000235585452088286
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.005629351828247309
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7246298486716114e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7246340323472396e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.172307616829235e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.011353629641234875
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 4.365778295323253e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 4.3657921196427196e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.382431946694851e-10,
"numeric_tolerance_ratio": 0.0001376422771007888
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0037216152995824814
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5795047147548757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.579504350956995e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6322416655772068e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492263289866969e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.185452315956354e-12,
"numeric_tolerance_ratio": 8.17360690293936e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006914579775184393
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1585754186380655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.158584877382964e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.428962719383602e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493387511232868e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.447291564678985e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5946865207515657e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.594685065560043e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4514255306884354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5181043383781798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5181052478728816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.072102508792652e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5752660803846084e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5752655346877873e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.442951166092543e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7124242478748783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.712418972805608e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.275069270282984e-11,
"numeric_tolerance_ratio": 5.260799777155296e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.8929631298524328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.8929516702191904e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1459633242338896e-10,
"numeric_tolerance_ratio": 0.00011426576708166119
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.3614365343528334e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.3614378076454159e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2715614306756342e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4222289792087395e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4222278878150973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0898436361148734e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.5869325074600056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.586930688470602e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8124881342346815e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0005169383366592228,
"latent_mse": 0.015044992789626122,
"scaled_prediction": 7.486693357350305e-05,
"scaled_weighted_latent": 0.00021789300080854446,
"total": 0.0002927599416580051
},
"gradients": {
"norms": {
"prediction": 0.001070561120286584,
"weighted_latent": 0.0012635241728276014,
"total": 0.002058987971395254
},
"prediction_weighted_latent_dot": 7.48418131024664e-07,
"prediction_weighted_latent_cosine": 0.5532854864566016,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0005865387502126396,
"latent_mse": 0.014186912216246128,
"scaled_prediction": 8.494699432048947e-05,
"scaled_weighted_latent": 0.0002054656360996887,
"total": 0.0002904126304201782
},
"gradients": {
"norms": {
"prediction": 0.001564753009006381,
"weighted_latent": 0.0012265188852325082,
"total": 0.0023790558334439993
},
"prediction_weighted_latent_dot": 8.535530469089281e-07,
"prediction_weighted_latent_cosine": 0.44474439346736716,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.029072362929582596
},
{
"step": 100,
"losses": {
"prediction_mse": 1.958247958100401e-05,
"latent_mse": 2.6493105906411074e-05,
"scaled_prediction": 2.8360832402540836e-06,
"scaled_weighted_latent": 3.836932762624201e-07,
"total": 3.219776544938213e-06
},
"gradients": {
"norms": {
"prediction": 2.248913369840011e-05,
"weighted_latent": 5.7423247199039906e-05,
"total": 5.3000701882410794e-05
},
"prediction_weighted_latent_dot": -4.970573952434165e-10,
"prediction_weighted_latent_cosine": -0.38489836452138804,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005795139586552978
},
{
"step": 200,
"losses": {
"prediction_mse": 2.0445553673198447e-05,
"latent_mse": 8.523401447746437e-06,
"scaled_prediction": 2.9610803267132724e-06,
"scaled_weighted_latent": 1.2344236210992676e-07,
"total": 3.0845226319797803e-06
},
"gradients": {
"norms": {
"prediction": 3.236097109038383e-05,
"weighted_latent": 3.241108788643032e-05,
"total": 2.005546775762923e-06
},
"prediction_weighted_latent_dot": -1.0468444999744975e-09,
"prediction_weighted_latent_cosine": -0.9980838347441332,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00010220007970929146
},
{
"step": 300,
"losses": {
"prediction_mse": 2.0447698261705227e-05,
"latent_mse": 8.498044735461008e-06,
"scaled_prediction": 2.961390919153928e-06,
"scaled_weighted_latent": 1.2307513941323123e-07,
"total": 3.084466015934595e-06
},
"gradients": {
"norms": {
"prediction": 3.2367966923629865e-05,
"weighted_latent": 3.2362138881580904e-05,
"total": 1.393101740632119e-07
},
"prediction_weighted_latent_dot": -1.0474870970611505e-09,
"prediction_weighted_latent_cosine": -0.9999908889103759,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 6.84225233271718e-06
},
{
"step": 400,
"losses": {
"prediction_mse": 2.0448296709218994e-05,
"latent_mse": 8.49935440783156e-06,
"scaled_prediction": 2.9614775485242717e-06,
"scaled_weighted_latent": 1.230940966934213e-07,
"total": 3.084571744693676e-06
},
"gradients": {
"norms": {
"prediction": 3.244182516937144e-05,
"weighted_latent": 3.2366053346777335e-05,
"total": 2.943585741377319e-06
},
"prediction_weighted_latent_dot": -1.045684538958369e-09,
"prediction_weighted_latent_cosine": -0.9958769066096964,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00014213190297596157
},
{
"step": 500,
"losses": {
"prediction_mse": 2.045622022706084e-05,
"latent_mse": 8.50646938488353e-06,
"scaled_prediction": 2.9626251034642337e-06,
"scaled_weighted_latent": 1.2319713960096124e-07,
"total": 3.0858222999086138e-06
},
"gradients": {
"norms": {
"prediction": 3.4209268051199615e-05,
"weighted_latent": 3.236785778426565e-05,
"total": 1.1325553714414127e-05
},
"prediction_weighted_latent_dot": -1.0448421017272835e-09,
"prediction_weighted_latent_cosine": -0.9436108475923198,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0004590976459439844
},
{
"step": 600,
"losses": {
"prediction_mse": 2.0449277144507505e-05,
"latent_mse": 8.499263458361384e-06,
"scaled_prediction": 2.9616194296977483e-06,
"scaled_weighted_latent": 1.230927892947875e-07,
"total": 3.0847122616251e-06
},
"gradients": {
"norms": {
"prediction": 3.28223904944025e-05,
"weighted_latent": 3.2365216611651704e-05,
"total": 5.815330951008946e-06
},
"prediction_weighted_latent_dot": -1.0454992427355592e-09,
"prediction_weighted_latent_cosine": -0.9841810453139609,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0001859236799646169
},
{
"step": 700,
"losses": {
"prediction_mse": 2.044854591076728e-05,
"latent_mse": 8.503768185619265e-06,
"scaled_prediction": 2.961513700938667e-06,
"scaled_weighted_latent": 1.23158031328785e-07,
"total": 3.084671789110871e-06
},
"gradients": {
"norms": {
"prediction": 3.274558548582718e-05,
"weighted_latent": 3.2373885915149e-05,
"total": 5.1326251195860095e-06
},
"prediction_weighted_latent_dot": -1.0469989319972228e-09,
"prediction_weighted_latent_cosine": -0.987639945387094,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.00017646911146584898
},
{
"step": 800,
"losses": {
"prediction_mse": 2.044757638941519e-05,
"latent_mse": 8.501372576574795e-06,
"scaled_prediction": 2.9613731840072433e-06,
"scaled_weighted_latent": 1.2312332842157048e-07,
"total": 3.0844964840071043e-06
},
"gradients": {
"norms": {
"prediction": 3.245994230383076e-05,
"weighted_latent": 3.2369178370572627e-05,
"total": 2.201377583332942e-06
},
"prediction_weighted_latent_dot": -1.0482827939028994e-09,
"prediction_weighted_latent_cosine": -0.9976978541911432,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 6.59990546409972e-05
},
{
"step": 900,
"losses": {
"prediction_mse": 2.045848304987885e-05,
"latent_mse": 8.527510544809047e-06,
"scaled_prediction": 2.9629527489305474e-06,
"scaled_weighted_latent": 1.2350187716947403e-07,
"total": 3.0864546261000214e-06
},
"gradients": {
"norms": {
"prediction": 3.636390465544537e-05,
"weighted_latent": 3.24251304846257e-05,
"total": 1.7051030226866715e-05
},
"prediction_weighted_latent_dot": -1.0414927809065944e-09,
"prediction_weighted_latent_cosine": -0.8832914388944041,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005175215192139149
},
{
"step": 1000,
"losses": {
"prediction_mse": 2.059229154838249e-05,
"latent_mse": 8.529103979526553e-06,
"scaled_prediction": 2.9823320346622495e-06,
"scaled_weighted_latent": 1.2352495559753152e-07,
"total": 3.1058571039466187e-06
},
"gradients": {
"norms": {
"prediction": 6.219880015123636e-05,
"weighted_latent": 3.2423569791717455e-05,
"total": 5.694122228305787e-05
},
"prediction_weighted_latent_dot": -8.38838110084339e-10,
"prediction_weighted_latent_cosine": -0.41594443457749386,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0016192415496334434
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "d5fe1daf76ecb831a7fd5993adb2ac04798fc23dccbf307d6a53d4b2f3928d4f"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "ba6b4ea59eee224296331a02da33b214f683ba0badba648e3cce0f5fc96d7ad8"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "ef74077bb905c12696391663b55c343802e51762e1bba59d3ce8e162f3dc5f65"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "9f258f22a24d87544a4479eb1bbe15290aa56fda9ac504e3b048d91ba72e6031"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "10150b04c1102445168f5c8fa27b63c71953fdb606de561e746ae1f62fa5d70f"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "ed4a08762deffdb1ab5010c569dcc919502f9d92929e2be17560541ff8652636"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "78bec8d77fdb13b0d5243041a503a5ef5a06e388c4d7d0160e14a0b823200c2f"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "cb98d9ea7c73430260da2b8d7d02fa6c1539927fe235fafd8dea39a6224555fa"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "9e8d42808a1d2dd4e7f6b7883a03bc120faf26841f07430dd12eb08377904313"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "f1d47374e9acd7f83376c995e0b75802f2dee0a379ebc93ba9fb3fc05ab660d7"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "0690360799a01c879c995c1ef5b193fd5d7267c4b7bda1db7d496ae067e7b34d"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "a0d56a6c0cb366f7ef9baa918d85d54904469a947720c2e1064b3d1c99352424"
}
],
"final_losses": {
"prediction_mse": 2.059229154838249e-05,
"latent_mse": 8.529103979526553e-06,
"scaled_prediction": 2.9823320346622495e-06,
"scaled_weighted_latent": 1.2352495559753152e-07,
"total": 3.1058571039466187e-06
},
"training_seconds": 3.331653399975039,
"evaluation_seconds": 0.5156276000197977
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 4,
"arm": "latent_only",
"status": "passed",
"files": {
"seed-4-latent_only/result.json": "c25ddf4e97d248b46a51fff346dc71b9fcffa5f3a72d78fc72bd5244d0d7333e",
"seed-4-latent_only/checkpoint.json": "2d931e11a6193b63dd4640446e85b1f406c5acc575a746081dcce32b3765c7c4",
"inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"seed-4-latent_only/diagnostics-arrays.npz": "14d0e799adb468983669afdb48f041e71ba151ee58eb2ed41ad25a12e37578a9"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.0978193283081055e-07,
"max_tolerance_ratio": 0.3144731636939299,
"worst_tolerance_index": [
0,
8
],
"actual_at_worst": 0.00018735229969024658,
"saved_at_worst": 0.0001876726746559143,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450094886356965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.3450114895240404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.996207215045689e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800791404908523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 5.800786675536074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.702096589973028e-05
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2923264503479004,
"saved_at_worst": 1.2923264503479004,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.48746048194662334,
"worst_tolerance_index": [
617,
0
],
"actual_at_worst": 0.0001264624297618866,
"saved_at_worst": 0.0001259688287973404,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.437260263832286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372990295290947
},
"reported": {
"count": 784,
"observable_mse": 2.437260263832286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372990295290947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.8886492398451082e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030165999196469784
},
"reported": {
"count": 580,
"observable_mse": 1.888649057946168e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003016600152477622
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001788720871244953
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.8339056370896287e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00303729809820652
},
"reported": {
"count": 112,
"observable_mse": 2.833907274180092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372983310371637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017858810755260759
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 5.4130203352542594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003164678579196334
},
"reported": {
"count": 92,
"observable_mse": 5.413021062850021e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031646783463656902
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017686010818346055
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5274550353060476e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211733050644398
},
"reported": {
"count": 98,
"observable_mse": 2.5274570361943915e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171675249934
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012327307636645429
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0186165784252807e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"reported": {
"count": 70,
"observable_mse": 2.0186189431115054e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211709767580032
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002817670465808772
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1840363842784427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171209588647
},
"reported": {
"count": 14,
"observable_mse": 2.1840449335286394e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221172373741865
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0008805219275269677
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.415066334535368e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221169812604785
},
"reported": {
"count": 14,
"observable_mse": 5.415061968960799e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.003698191444345532
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493371140328236e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220667831599712
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.008045871341972934
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9936096578021534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022065619006753
},
"reported": {
"count": 70,
"observable_mse": 1.9936062017222866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220558401197195
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.778887033462524e-09,
"numeric_tolerance_ratio": 0.007509480187709456
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.3793707441654988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022068180143833
},
"reported": {
"count": 14,
"observable_mse": 2.3793658328941092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009118654187569325
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.497939855558798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022070275619626
},
"reported": {
"count": 14,
"observable_mse": 5.497950769495219e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220600310713053
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.007867071950464247
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5550732971169055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836770940572023
},
"reported": {
"count": 98,
"observable_mse": 2.55507566180313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836845446377993
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.005651364435865696
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.939376124937553e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836775597184896
},
"reported": {
"count": 70,
"observable_mse": 1.9393763068364933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836838461458683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004768338995395631
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.679098724911455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836782582104206
},
"reported": {
"count": 14,
"observable_mse": 2.679110002645757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836875714361668
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.007064203922984993
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.509533730219118e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836743000894785
},
"reported": {
"count": 14,
"observable_mse": 5.509537731995806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003183686174452305
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009006860956200402
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5143033781205304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363468684256077
},
"reported": {
"count": 98,
"observable_mse": 2.514301559131127e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634756691753864
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.005226873219612861
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949908801179845e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588928636163473
},
"reported": {
"count": 72,
"observable_mse": 1.949909346876666e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588993828743696
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004880086177358148
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.875245627365075e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634670544415712
},
"reported": {
"count": 14,
"observable_mse": 2.8752372600138187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363472642377019
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004181499522790897
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.4795717005617917e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033907967153936625
},
"reported": {
"count": 12,
"observable_mse": 5.4795644246041775e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003390809055417776
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.00921529390986458
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5130846552201547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 98,
"observable_mse": 2.5130850190180354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750871662050486
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0031334013256818006
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949693978531286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003370406571775675
},
"reported": {
"count": 72,
"observable_mse": 1.949694888025988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033704107627272606
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.003134497256772981
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750897273421288
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.752088665962219e-09,
"numeric_tolerance_ratio": 0.005048256724707581
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.473628334584646e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003403007285669446
},
"reported": {
"count": 12,
"observable_mse": 5.4736312449676916e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034030096139758825
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0017371519558643553
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509041734039783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103164955973625
},
"reported": {
"count": 98,
"observable_mse": 2.5509052647976205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103166352957487
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0010661422012763408
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9929024347220547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031647231429815
},
"reported": {
"count": 70,
"observable_mse": 1.9929013433284126e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031661201268435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.001066142220220661
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.406693783996161e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103160299360752
},
"reported": {
"count": 14,
"observable_mse": 2.406714338576421e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031658872962
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004264568956659927
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.485124347615056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103169146925211
},
"reported": {
"count": 14,
"observable_mse": 5.485117799253203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031689140945673
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001776903321481438
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386959456838667e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.3563443107832427e-07
},
"reported": {
"count": 98,
"observable_mse": 2.8386977646732703e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.3200118448585272e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.6332465924715507e-09,
"numeric_tolerance_ratio": 0.003633162302675786
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450111257261597e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.3472955490433378e-07
},
"reported": {
"count": 84,
"observable_mse": 2.3450154913007282e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.3212984956444416e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.599705339889624e-09,
"numeric_tolerance_ratio": 0.0025996449943694777
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800804137834348e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.4099242068587046e-07
},
"reported": {
"count": 14,
"observable_mse": 5.800792860100046e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 2.312278155613967e-07
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.764605124473746e-09,
"numeric_tolerance_ratio": 0.009764379344863126
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492266927845776e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384734462946653
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006583761406554621
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.54845256981207e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003532622940838337
},
"reported": {
"count": 72,
"observable_mse": 8.548483492631931e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035326124634593725
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.0077423180429610685
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4730835573573131e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003438472282141447
},
"reported": {
"count": 14,
"observable_mse": 1.4730812836205587e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.683411240577698e-09,
"numeric_tolerance_ratio": 0.005717477010955329
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.987682041246444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080525808036327
},
"reported": {
"count": 12,
"observable_mse": 4.987671127310023e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080549091100693
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0018178454519917982
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2935075759887695,
"saved_at_worst": 1.2935075759887695,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.5794924900882756,
"worst_tolerance_index": [
5875,
12
],
"actual_at_worst": 0.0005421936511993408,
"saved_at_worst": 0.0005428045988082886,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.416123970760964e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005598969757556915
},
"reported": {
"count": 6272,
"observable_mse": 2.4161243345588446e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005598969291895628
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.656612873077393e-10,
"numeric_tolerance_ratio": 0.0002985205487580974
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.8748192815110087e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055619413033127785
},
"reported": {
"count": 4092,
"observable_mse": 1.874819463409949e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0055619413033127785
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-12,
"numeric_tolerance_ratio": 1.8155855084970412e-06
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.7032006983063184e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005598968360573053
},
"reported": {
"count": 896,
"observable_mse": 2.7032017896999605e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005598968360573053
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0884513385212222e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.9408889279002324e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005715375766158104
},
"reported": {
"count": 1284,
"observable_mse": 3.940889655495994e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.005715375766158104
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.247396424584503e-06
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2881712913513184,
"saved_at_worst": 1.2881712913513184,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4369506557359455,
"worst_tolerance_index": [
2489,
7
],
"actual_at_worst": 0.0009123608469963074,
"saved_at_worst": 0.0009128376841545105,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.416784238652326e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010839683003723621
},
"reported": {
"count": 3136,
"observable_mse": 3.416786785237491e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010839683003723621
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5379136551252936e-05
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 2.91789820039412e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01080263126641512
},
"reported": {
"count": 1582,
"observable_mse": 2.9179005650803447e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01080263126641512
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.357806380040942e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.484251647023484e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010839683935046196
},
"reported": {
"count": 448,
"observable_mse": 3.484255285002291e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010839683935046196
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6253471720475385e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 4.1030518332263455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01089246105402708
},
"reported": {
"count": 1106,
"observable_mse": 4.1030543798115104e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01089246105402708
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.546585164964199e-11,
"numeric_tolerance_ratio": 2.5361790842645216e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2869694232940674,
"saved_at_worst": 1.2869694232940674,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4155178327569591,
"worst_tolerance_index": [
2367,
8
],
"actual_at_worst": -0.0002209991216659546,
"saved_at_worst": -0.00022057443857192993,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 8.863356924848631e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020777098834514618
},
"reported": {
"count": 3136,
"observable_mse": 8.863364200806245e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02077709697186947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006052049518944008
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 7.996758358785883e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02014041878283024
},
"reported": {
"count": 1001,
"observable_mse": 7.996763451956213e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02014041878283024
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.093170329928398e-11,
"numeric_tolerance_ratio": 5.052764566908406e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 8.91760500962846e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02077709697186947
},
"reported": {
"count": 448,
"observable_mse": 8.91761010279879e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020777098834514618
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006052049152670997
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 9.363156277686357e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.021145818755030632
},
"reported": {
"count": 1687,
"observable_mse": 9.363165736431256e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.021145818755030632
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.371002647533067e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2679539918899536,
"saved_at_worst": 1.2679539918899536,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.42896605801972054,
"worst_tolerance_index": [
3103,
0
],
"actual_at_worst": -0.0004641897976398468,
"saved_at_worst": -0.00046463869512081146,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0003656770568341017,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043556950986385345
},
"reported": {
"count": 3136,
"observable_mse": 0.0003656771732494235,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043556950986385345
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-10,
"numeric_tolerance_ratio": 0.00011230845788576787
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.0003249439469072968,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04104834794998169
},
"reported": {
"count": 407,
"observable_mse": 0.0003249441215302795,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04104834794998169
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016912729084534847
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0003670162695925683,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043556950986385345
},
"reported": {
"count": 448,
"observable_mse": 0.000367016444215551,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043556954711675644
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.0006955754520616506
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.00037268202868290246,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043989527970552444
},
"reported": {
"count": 2281,
"observable_mse": 0.00037268217420205474,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.043989527970552444
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014029076553178296
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006257921922951937
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5511306375847198e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5511330022709444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3586689464189724e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006255257874727249
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5392424504389055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5392444513272494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9958204678516046e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006481279619038105
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5999313947977498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5999332137871534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8142724164315183e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006476297043263912
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.6249348593410105e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.624933586048428e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2699590243860832e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.021607279777526855
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.766223668819293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.7661858338397e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.7834979593753815e-10,
"numeric_tolerance_ratio": 0.00037580701583076647
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.019717823714017868
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.143248901935294e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.143239443190396e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.40099235142008e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020352382212877274
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.072931475704536e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.072944572428241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00013004741886984787
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012590174563229084
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.74880874005612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.748832386918366e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-10,
"numeric_tolerance_ratio": 0.000235585452088286
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 3.657481784102856e-07
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.838700129359495e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.838699583662674e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.4415213662008435e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0059067197144031525
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7997512006550096e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.7997439246973954e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.255643669898858e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0037216152995824814
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5795047147548757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.579504350956995e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6322416655772068e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492263289866969e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.185452315956354e-12,
"numeric_tolerance_ratio": 8.17360690293936e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006914579775184393
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1585754186380655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.158584877382964e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.428962719383602e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493387511232868e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.447291564678985e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5946865207515657e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.594685065560043e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4514255306884354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5181043383781798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5181052478728816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.072102508792652e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509087208774872e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5509103579679504e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6329250178494538e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.676446456462145e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.6764359063236043e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0550138540565968e-10,
"numeric_tolerance_ratio": 0.00010521977143134568
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.8929631298524328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.8929516702191904e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1459633242338896e-10,
"numeric_tolerance_ratio": 0.00011426576708166119
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.3614365343528334e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.3614378076454159e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2715614306756342e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4222289792087395e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4222278878150973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0898436361148734e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.5869325074600056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.586930688470602e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8124881342346815e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 1000,
"optimizer_initial_state": {
"type": "Adam",
"parameter_shape": [
10,
11
],
"learning_rate": 0.003,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.0,
"foreach": false,
"fused": false,
"fresh": true,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 0.0005169383366592228,
"latent_mse": 0.015044992789626122,
"scaled_prediction": 7.486693357350305e-05,
"scaled_weighted_latent": 0.00021789300080854446,
"total": 0.00021789300080854446
},
"gradients": {
"norms": {
"prediction": 0.001070561120286584,
"weighted_latent": 0.0012635241728276014,
"total": 0.0012635241728276014
},
"prediction_weighted_latent_dot": 7.48418131024664e-07,
"prediction_weighted_latent_cosine": 0.5532854864566016,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
},
{
"step": 1,
"losses": {
"prediction_mse": 0.0005966797471046448,
"latent_mse": 0.014187210239470005,
"scaled_prediction": 8.641569002065808e-05,
"scaled_weighted_latent": 0.0002054699434665963,
"total": 0.0002054699434665963
},
"gradients": {
"norms": {
"prediction": 0.0015940532321110368,
"weighted_latent": 0.0012265965342521667,
"total": 0.0012265965342521667
},
"prediction_weighted_latent_dot": 8.696779332240112e-07,
"prediction_weighted_latent_cosine": 0.4447888555205473,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.03141213580965996
},
{
"step": 100,
"losses": {
"prediction_mse": 2.1970061425236054e-05,
"latent_mse": 1.11036715679802e-05,
"scaled_prediction": 3.181871079505072e-06,
"scaled_weighted_latent": 1.6081179410321056e-07,
"total": 1.6081179410321056e-07
},
"gradients": {
"norms": {
"prediction": 0.00014432429452426732,
"weighted_latent": 3.859043135889806e-05,
"total": 3.859043135889806e-05
},
"prediction_weighted_latent_dot": 1.7062163903958094e-09,
"prediction_weighted_latent_cosine": 0.3063479886041389,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0005903447163291276
},
{
"step": 200,
"losses": {
"prediction_mse": 2.346691871935036e-05,
"latent_mse": 1.3682244226487228e-10,
"scaled_prediction": 3.3986573271249654e-06,
"scaled_weighted_latent": 1.981566386671796e-12,
"total": 1.981566386671796e-12
},
"gradients": {
"norms": {
"prediction": 0.0001162839907919988,
"weighted_latent": 1.2867796783666563e-07,
"total": 1.2867796783666563e-07
},
"prediction_weighted_latent_dot": 1.2082485185971326e-11,
"prediction_weighted_latent_cosine": 0.8074806977710687,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.09139659293578e-06
},
{
"step": 300,
"losses": {
"prediction_mse": 2.3450125809176825e-05,
"latent_mse": 5.86610092640806e-14,
"scaled_prediction": 3.396225110918749e-06,
"scaled_weighted_latent": 8.495732520026752e-16,
"total": 8.495732520026752e-16
},
"gradients": {
"norms": {
"prediction": 0.0001155613936134614,
"weighted_latent": 2.045127178718076e-09,
"total": 2.045127178718076e-09
},
"prediction_weighted_latent_dot": 4.812350774222474e-15,
"prediction_weighted_latent_cosine": 0.020362175901058977,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.1458557480636955e-07
},
{
"step": 400,
"losses": {
"prediction_mse": 2.3450151275028475e-05,
"latent_mse": 6.174448024262938e-14,
"scaled_prediction": 3.3962289762712317e-06,
"scaled_weighted_latent": 8.942304701082572e-16,
"total": 8.942304701082572e-16
},
"gradients": {
"norms": {
"prediction": 0.00011556194658624008,
"weighted_latent": 2.1368944391753075e-09,
"total": 2.1368944391753075e-09
},
"prediction_weighted_latent_dot": 7.334172840171762e-15,
"prediction_weighted_latent_cosine": 0.029699779355648236,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 1.903740951547661e-07
},
{
"step": 500,
"losses": {
"prediction_mse": 2.345011853321921e-05,
"latent_mse": 4.6259938696509295e-14,
"scaled_prediction": 3.3962242014240474e-06,
"scaled_weighted_latent": 6.699715622404255e-16,
"total": 6.699715622404255e-16
},
"gradients": {
"norms": {
"prediction": 0.00011556225217645988,
"weighted_latent": 1.8800572210153632e-09,
"total": 1.8800572210153632e-09
},
"prediction_weighted_latent_dot": 7.533627923426468e-15,
"prediction_weighted_latent_cosine": 0.034675050513547474,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.09897160377659e-07
},
{
"step": 600,
"losses": {
"prediction_mse": 2.3450134904123843e-05,
"latent_mse": 5.471406273470543e-14,
"scaled_prediction": 3.3962264751608018e-06,
"scaled_weighted_latent": 7.924105982825083e-16,
"total": 7.924105982825083e-16
},
"gradients": {
"norms": {
"prediction": 0.0001155606732936576,
"weighted_latent": 2.391824294178946e-09,
"total": 2.391824294178946e-09
},
"prediction_weighted_latent_dot": -8.797817820939934e-16,
"prediction_weighted_latent_cosine": -0.003182992595752107,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 2.5450245288993756e-07
},
{
"step": 700,
"losses": {
"prediction_mse": 2.345011853321921e-05,
"latent_mse": 7.094315640585741e-14,
"scaled_prediction": 3.3962242014240474e-06,
"scaled_weighted_latent": 1.0274526590853608e-15,
"total": 1.0274526590853608e-15
},
"gradients": {
"norms": {
"prediction": 0.00011556035315152258,
"weighted_latent": 3.254916558503851e-09,
"total": 3.254916558503851e-09
},
"prediction_weighted_latent_dot": 3.4637555258227757e-15,
"prediction_weighted_latent_cosine": 0.009208703960344337,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.699499752154225e-07
},
{
"step": 800,
"losses": {
"prediction_mse": 2.3450140361092053e-05,
"latent_mse": 4.264028230982139e-14,
"scaled_prediction": 3.3962273846555036e-06,
"scaled_weighted_latent": 6.175489048930937e-16,
"total": 6.175489048930937e-16
},
"gradients": {
"norms": {
"prediction": 0.00011556156096048653,
"weighted_latent": 2.075037919269107e-09,
"total": 2.075037919269107e-09
},
"prediction_weighted_latent_dot": -1.0912143347393492e-15,
"prediction_weighted_latent_cosine": -0.004550620569282825,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 3.335407825488801e-07
},
{
"step": 900,
"losses": {
"prediction_mse": 2.345009306736756e-05,
"latent_mse": 2.7154482072741688e-14,
"scaled_prediction": 3.3962205634452403e-06,
"scaled_weighted_latent": 3.932718122866757e-16,
"total": 3.932718122866757e-16
},
"gradients": {
"norms": {
"prediction": 0.0001155607751570642,
"weighted_latent": 1.4901179179460655e-09,
"total": 1.4901179179460655e-09
},
"prediction_weighted_latent_dot": 4.4029780567485764e-15,
"prediction_weighted_latent_cosine": 0.02556909977134831,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 5.495569439517567e-07
},
{
"step": 1000,
"losses": {
"prediction_mse": 2.3450154913007282e-05,
"latent_mse": 5.388426521386186e-14,
"scaled_prediction": 3.3962294310185825e-06,
"scaled_weighted_latent": 7.803928418873051e-16,
"total": 7.803928418873051e-16
},
"gradients": {
"norms": {
"prediction": 0.00011556116078281775,
"weighted_latent": 2.6767299488028584e-09,
"total": 2.6767299488028584e-09
},
"prediction_weighted_latent_dot": 1.5839785470132643e-14,
"prediction_weighted_latent_cosine": 0.05120741368883312,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 9.743667988004745e-07
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "f699effcd25959f7680c911e7e747dadfe1905be0a36d1db90898b83d30d32e2"
},
{
"step": 1,
"path": "checkpoints/step-0001.json",
"sha256": "16d6290bd18f6fc09a6d359f42c2850fd7be6d053ff3733c29ed632302aed6ac"
},
{
"step": 100,
"path": "checkpoints/step-0100.json",
"sha256": "79468a7b7bb60cb31ae067bd466365ae6ce7ae479a6c5c974029a0277487ea75"
},
{
"step": 200,
"path": "checkpoints/step-0200.json",
"sha256": "67590f7564bf109fc28336dca4bb0714de4e6b7975593730f38566feee20fdcc"
},
{
"step": 300,
"path": "checkpoints/step-0300.json",
"sha256": "d2d1d7f10f51fd7867f9870527fd6b4a75abf7f76da66c4a162b05fbfc7b3a74"
},
{
"step": 400,
"path": "checkpoints/step-0400.json",
"sha256": "96b08ccd99469c3263897cd45890578b23fbe119971ba0f9636425b26ac97a06"
},
{
"step": 500,
"path": "checkpoints/step-0500.json",
"sha256": "b5bac88343b6003c810ddaeee3592e4f0f114dc48d28e324ad13a336fc8b57b0"
},
{
"step": 600,
"path": "checkpoints/step-0600.json",
"sha256": "e75779ee7f9a7ead20804c9d3654a7014174f133e1fefde530cd3552af26f5e4"
},
{
"step": 700,
"path": "checkpoints/step-0700.json",
"sha256": "7dae43d65fefd08534539ce218529e26f7e99234942b39ee4122e3e31d921e9f"
},
{
"step": 800,
"path": "checkpoints/step-0800.json",
"sha256": "db64aa038ad79127161d8cf6c8bebea01282a7973c6b4b473129af617d6374f5"
},
{
"step": 900,
"path": "checkpoints/step-0900.json",
"sha256": "59723d6858c9144b10a158ec319c65303b53ff752beb7888c02a7910687653f8"
},
{
"step": 1000,
"path": "checkpoints/step-1000.json",
"sha256": "2d931e11a6193b63dd4640446e85b1f406c5acc575a746081dcce32b3765c7c4"
}
],
"final_losses": {
"prediction_mse": 2.3450154913007282e-05,
"latent_mse": 5.388426521386186e-14,
"scaled_prediction": 3.3962294310185825e-06,
"scaled_weighted_latent": 7.803928418873051e-16,
"total": 7.803928418873051e-16
},
"training_seconds": 2.835771100013517,
"evaluation_seconds": 0.4971197000122629
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
},
{
"seed": 4,
"arm": "refit",
"status": "passed",
"files": {
"seed-4-refit/result.json": "bbeefd997166ae9237853a96b43f56fb313eddeba42dfb979793b79d0007ae88",
"seed-4-refit/checkpoint.json": "54d0f37cf21d2f3971d6eede53623bef7794973f5523f1f4b98bab3c2bb4a858",
"inputs/seed-4-fourier/checkpoint.json": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"inputs/seed-4-fourier/result.json": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"seed-4-refit/diagnostics-arrays.npz": "e1c92542d749f49d0bb2b5de987d06fb43bcfc8239d0efffa527ec14fff421f8"
},
"provenance": {
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"parent_source_sha256": "81c8691236213584088caa27f593656f6be36e18786ccfb992e3a8c3540334df",
"design_file_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"parent_result_sha256": "d53aab6e2527e28c7a13d98a674c389f41b5e852d8a4f7116f1461633ef5fbc9",
"parent_checkpoint_sha256": "113b41bfec83ed51c92dddb71af13906d2920172dc8fac636dfb86005f40d454",
"parent_arrays_sha256": "6b72cf3fd7037dfe7ec871c7a291eb2e7d71d2c8f6ea8005cbc0316cda513e57",
"recorded_refit_sha256": "4205cf42eb4ad75121041f586c303de89be03f0259e36ffcc423d34b7080e314"
},
"frozen_tensor_isolation": "passed",
"encoding": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"cases": [
{
"set": "reconstruction",
"count": 98,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
98,
10
],
"compared_scalars": 980,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.29099440574646,
"saved_at_worst": 1.29099440574646,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
98,
16
],
"compared_scalars": 1568,
"outside_tolerance": 0,
"max_absolute_error": 4.0978193283081055e-07,
"max_tolerance_ratio": 0.3144731636939299,
"worst_tolerance_index": [
0,
8
],
"actual_at_worst": 0.00018735229969024658,
"saved_at_worst": 0.0001876726746559143,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 98,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
},
{
"case": "seen",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450094886356965e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 84,
"observable_mse": 2.3450114895240404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.996207215045689e-05
},
"comparisons": 6
},
{
"case": "held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800791404908523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 14,
"observable_mse": 5.800786675536074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.702096589973028e-05
},
"comparisons": 6
}
],
"saved_diagnostic_differences": []
},
{
"set": "one_step",
"count": 784,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
784,
10
],
"compared_scalars": 7840,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2923264503479004,
"saved_at_worst": 1.2923264503479004,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
784,
16
],
"compared_scalars": 12544,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.37291832992681323,
"worst_tolerance_index": [
343,
9
],
"actual_at_worst": 0.001188725233078003,
"saved_at_worst": 0.0011883080005645752,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 784,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 784,
"observable_mse": 2.437259718135465e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372990295290947
},
"reported": {
"count": 784,
"observable_mse": 2.437259718135465e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372990295290947
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 580,
"observable_mse": 1.8886488760472275e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030165999196469784
},
"reported": {
"count": 580,
"observable_mse": 1.8886488760472275e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003016600152477622
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001788720871244953
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 112,
"observable_mse": 2.8339039999991655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.00303729809820652
},
"reported": {
"count": 112,
"observable_mse": 2.833904727594927e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030372983310371637
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017858810755260759
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 92,
"observable_mse": 5.4130203352542594e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003164678579196334
},
"reported": {
"count": 92,
"observable_mse": 5.413021062850021e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031646783463656902
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.00017686010818346055
},
"comparisons": 7
},
{
"case": "action_X/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5274550353060476e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211733050644398
},
"reported": {
"count": 98,
"observable_mse": 2.5274570361943915e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171675249934
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6298145055770874e-09,
"numeric_tolerance_ratio": 0.0012327307636645429
},
"comparisons": 7
},
{
"case": "action_X/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 2.0186165784252807e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"reported": {
"count": 70,
"observable_mse": 2.0186189431115054e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211709767580032
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.725290298461914e-09,
"numeric_tolerance_ratio": 0.002817670465808772
},
"comparisons": 7
},
{
"case": "action_X/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.1840363842784427e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221171209588647
},
"reported": {
"count": 14,
"observable_mse": 2.1840449335286394e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221172373741865
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1641532182693481e-09,
"numeric_tolerance_ratio": 0.0008805219275269677
},
"comparisons": 7
},
{
"case": "action_X/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.415066334535368e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003221169812604785
},
"reported": {
"count": 14,
"observable_mse": 5.415061968960799e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0032211747020483017
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.889443516731262e-09,
"numeric_tolerance_ratio": 0.003698191444345532
},
"comparisons": 7
},
{
"case": "action_Y/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493371140328236e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220667831599712
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.008045871341972934
},
"comparisons": 7
},
{
"case": "action_Y/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9936096578021534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022065619006753
},
"reported": {
"count": 70,
"observable_mse": 1.9936062017222866e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220558401197195
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.778887033462524e-09,
"numeric_tolerance_ratio": 0.007509480187709456
},
"comparisons": 7
},
{
"case": "action_Y/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.3793707441654988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022068180143833
},
"reported": {
"count": 14,
"observable_mse": 2.3793658328941092e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022056305781007
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009118654187569325
},
"comparisons": 7
},
{
"case": "action_Y/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.497939855558798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003022070275619626
},
"reported": {
"count": 14,
"observable_mse": 5.497950769495219e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0030220600310713053
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0244548320770264e-08,
"numeric_tolerance_ratio": 0.007867071950464247
},
"comparisons": 7
},
{
"case": "action_Xinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5550732971169055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836770940572023
},
"reported": {
"count": 98,
"observable_mse": 2.55507566180313e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836845446377993
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.450580596923828e-09,
"numeric_tolerance_ratio": 0.005651364435865696
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.939376124937553e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836775597184896
},
"reported": {
"count": 70,
"observable_mse": 1.9393763068364933e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836838461458683
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.28642737865448e-09,
"numeric_tolerance_ratio": 0.004768338995395631
},
"comparisons": 7
},
{
"case": "action_Xinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.679098724911455e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836782582104206
},
"reported": {
"count": 14,
"observable_mse": 2.679110002645757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836875714361668
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-09,
"numeric_tolerance_ratio": 0.007064203922984993
},
"comparisons": 7
},
{
"case": "action_Xinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.509533730219118e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031836743000894785
},
"reported": {
"count": 14,
"observable_mse": 5.509537731995806e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003183686174452305
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1874362826347351e-08,
"numeric_tolerance_ratio": 0.009006860956200402
},
"comparisons": 7
},
{
"case": "action_R/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5143033781205304e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363468684256077
},
"reported": {
"count": 98,
"observable_mse": 2.514301559131127e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634756691753864
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.984919309616089e-09,
"numeric_tolerance_ratio": 0.005226873219612861
},
"comparisons": 7
},
{
"case": "action_R/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949908801179845e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588928636163473
},
"reported": {
"count": 72,
"observable_mse": 1.949909346876666e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033588993828743696
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.51925802230835e-09,
"numeric_tolerance_ratio": 0.004880086177358148
},
"comparisons": 7
},
{
"case": "action_R/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.875245627365075e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033634670544415712
},
"reported": {
"count": 14,
"observable_mse": 2.8752372600138187e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003363472642377019
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004181499522790897
},
"comparisons": 7
},
{
"case": "action_R/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.4795717005617917e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033907967153936625
},
"reported": {
"count": 12,
"observable_mse": 5.4795644246041775e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003390809055417776
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.234002411365509e-08,
"numeric_tolerance_ratio": 0.00921529390986458
},
"comparisons": 7
},
{
"case": "action_Rinv/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5130846552201547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 98,
"observable_mse": 2.5130850190180354e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750871662050486
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.0031334013256818006
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 1.949693978531286e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003370406571775675
},
"reported": {
"count": 72,
"observable_mse": 1.949694888025988e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033704107627272606
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.190951585769653e-09,
"numeric_tolerance_ratio": 0.003134497256772981
},
"comparisons": 7
},
{
"case": "action_Rinv/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003375082975253463
},
"reported": {
"count": 14,
"observable_mse": 2.872911500162445e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0033750897273421288
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 6.752088665962219e-09,
"numeric_tolerance_ratio": 0.005048256724707581
},
"comparisons": 7
},
{
"case": "action_Rinv/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 5.473628334584646e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003403007285669446
},
"reported": {
"count": 12,
"observable_mse": 5.4736312449676916e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034030096139758825
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0017371519558643553
},
"comparisons": 7
},
{
"case": "action_S/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509041734039783e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103164955973625
},
"reported": {
"count": 98,
"observable_mse": 2.5509052647976205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103166352957487
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.0010661422012763408
},
"comparisons": 7
},
{
"case": "action_S/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 70,
"observable_mse": 1.9929024347220547e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031647231429815
},
"reported": {
"count": 70,
"observable_mse": 1.9929013433284126e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031661201268435
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3969838619232178e-09,
"numeric_tolerance_ratio": 0.001066142220220661
},
"comparisons": 7
},
{
"case": "action_S/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 2.406693783996161e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103160299360752
},
"reported": {
"count": 14,
"observable_mse": 2.406714338576421e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031658872962
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.587935447692871e-09,
"numeric_tolerance_ratio": 0.004264568956659927
},
"comparisons": 7
},
{
"case": "action_S/seen_enters_held",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.485124347615056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003103169146925211
},
"reported": {
"count": 14,
"observable_mse": 5.485117799253203e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0031031689140945673
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-10,
"numeric_tolerance_ratio": 0.0001776903321481438
},
"comparisons": 7
},
{
"case": "action_H/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.838693035300821e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.229526359206479e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6276809331726797e-06
},
"comparisons": 7
},
{
"case": "action_H/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 84,
"observable_mse": 2.3450098524335772e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"reported": {
"count": 84,
"observable_mse": 2.3450114895240404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 5.613619272490454e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.6370904631912708e-11,
"numeric_tolerance_ratio": 1.6332604486737455e-05
},
"comparisons": 7
},
{
"case": "action_H/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 5.800791404908523e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360026644082518e-16
},
"reported": {
"count": 14,
"observable_mse": 5.800786675536074e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 1.5360023997104558e-16
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.729372449219227e-11,
"numeric_tolerance_ratio": 4.702096589973028e-05
},
"comparisons": 7
},
{
"case": "action_H/seen_enters_held",
"status": "passed",
"actual": {
"count": 0,
"status": "empty"
},
"reported": {
"count": 0,
"status": "empty"
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 1
},
{
"case": "action_P/all",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492266927845776e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384734462946653
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.847564458847046e-09,
"numeric_tolerance_ratio": 0.006583761406554621
},
"comparisons": 7
},
{
"case": "action_P/seen_stays_in_support",
"status": "passed",
"actual": {
"count": 72,
"observable_mse": 8.54845256981207e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003532622940838337
},
"reported": {
"count": 72,
"observable_mse": 8.548483492631931e-06,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0035326124634593725
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0477378964424133e-08,
"numeric_tolerance_ratio": 0.0077423180429610685
},
"comparisons": 7
},
{
"case": "action_P/held_initial",
"status": "passed",
"actual": {
"count": 14,
"observable_mse": 1.4730835573573131e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.003438472282141447
},
"reported": {
"count": 14,
"observable_mse": 1.4730812836205587e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0034384645987302065
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.683411240577698e-09,
"numeric_tolerance_ratio": 0.005717477010955329
},
"comparisons": 7
},
{
"case": "action_P/seen_enters_held",
"status": "passed",
"actual": {
"count": 12,
"observable_mse": 4.987682041246444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080525808036327
},
"reported": {
"count": 12,
"observable_mse": 4.987671127310023e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.0028080549091100693
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3283064365386963e-09,
"numeric_tolerance_ratio": 0.0018178454519917982
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_2",
"count": 6272,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
6272,
10
],
"compared_scalars": 62720,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2935075759887695,
"saved_at_worst": 1.2935075759887695,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
6272,
16
],
"compared_scalars": 100352,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.5794924900882756,
"worst_tolerance_index": [
5875,
12
],
"actual_at_worst": 0.0005421936511993408,
"saved_at_worst": 0.0005428045988082886,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 6272,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 6272,
"observable_mse": 2.4130684323608875e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005596357863396406
},
"reported": {
"count": 6272,
"observable_mse": 2.413069159956649e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005596357863396406
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.258442490460377e-06
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 4092,
"observable_mse": 1.8717970306170173e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005558918230235577
},
"reported": {
"count": 4092,
"observable_mse": 1.8717970306170173e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005558918230235577
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 0.0,
"numeric_tolerance_ratio": 0.0
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 896,
"observable_mse": 2.6992713173967786e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005596356466412544
},
"reported": {
"count": 896,
"observable_mse": 2.6992727725883015e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005596356466412544
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4512741380702305e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1284,
"observable_mse": 3.938337249564938e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.005714038852602243
},
"reported": {
"count": 1284,
"observable_mse": 3.938339068554342e-05,
"exact_state_accuracy": 0.9999999403953552,
"per_field_accuracy": {
"x": 0.9999999403953552,
"y": 0.9999999403953552,
"h": 0.9999999403953552
},
"normalized_latent_rmse": 0.005714038852602243
},
"differences": [],
"max_errors": {
"accuracy_abs": 5.960464477539063e-08,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8118537092960308e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_4",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2876759767532349,
"saved_at_worst": 1.2876759767532349,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 7.152557373046875e-07,
"max_tolerance_ratio": 0.4632957891476394,
"worst_tolerance_index": [
655,
8
],
"actual_at_worst": 0.0014174729585647583,
"saved_at_worst": 0.00141800194978714,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 3.346535959281027e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010823823511600494
},
"reported": {
"count": 3136,
"observable_mse": 3.3465381420683116e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.01082382258027792
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.313225746154785e-10,
"numeric_tolerance_ratio": 0.0004472390076438352
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1582,
"observable_mse": 2.8475007638917305e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010785087011754513
},
"reported": {
"count": 1582,
"observable_mse": 2.8475038561737165e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010785087011754513
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.092281986027956e-11,
"numeric_tolerance_ratio": 3.0835017030380373e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 3.412026126170531e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010823823511600494
},
"reported": {
"count": 448,
"observable_mse": 3.412029764149338e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010823823511600494
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-11,
"numeric_tolerance_ratio": 3.6256081242586016e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1106,
"observable_mse": 4.033817822346464e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010878991335630417
},
"reported": {
"count": 1106,
"observable_mse": 4.033818913740106e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.010878991335630417
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0870088452879884e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_8",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2870361804962158,
"saved_at_worst": 1.2870361804962158,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.4524213331222226,
"worst_tolerance_index": [
206,
7
],
"actual_at_worst": -0.00037545710802078247,
"saved_at_worst": -0.00037498772144317627,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 8.535668166587129e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020744062960147858
},
"reported": {
"count": 3136,
"observable_mse": 8.535675442544743e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.02074406109750271
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.862645149230957e-09,
"numeric_tolerance_ratio": 0.0006058552717949994
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 1001,
"observable_mse": 7.651780470041558e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020098954439163208
},
"reported": {
"count": 1001,
"observable_mse": 7.651787745999172e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020098954439163208
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-11,
"numeric_tolerance_ratio": 7.220706302182925e-05
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 8.586674084654078e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020744062960147858
},
"reported": {
"count": 448,
"observable_mse": 8.586679905420169e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.020744062960147858
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.820766091346741e-11,
"numeric_tolerance_ratio": 5.771210553655716e-05
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 1687,
"observable_mse": 9.046589548233896e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.021117528900504112
},
"reported": {
"count": 1687,
"observable_mse": 9.046597551787272e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.021117528900504112
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.003553375601768e-11,
"numeric_tolerance_ratio": 7.931797594898488e-05
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
},
{
"set": "composition_16",
"count": 3136,
"status": "passed",
"continuous": {
"latent": {
"status": "passed",
"shape": [
3136,
10
],
"compared_scalars": 31360,
"outside_tolerance": 0,
"max_absolute_error": 0.0,
"max_tolerance_ratio": 0.0,
"worst_tolerance_index": [
0,
0
],
"actual_at_worst": 1.2679535150527954,
"saved_at_worst": 1.2679535150527954,
"witnesses": []
},
"logits": {
"status": "passed",
"shape": [
3136,
16
],
"compared_scalars": 50176,
"outside_tolerance": 0,
"max_absolute_error": 5.960464477539062e-07,
"max_tolerance_ratio": 0.42896605801972054,
"worst_tolerance_index": [
3103,
0
],
"actual_at_worst": -0.0004641897976398468,
"saved_at_worst": -0.00046463869512081146,
"witnesses": []
}
},
"categorical": {
"status": "passed",
"compared_predictions": 3136,
"differing_predictions": 0,
"cases": []
},
"metric_comparisons": [
{
"case": "all",
"status": "passed",
"actual": {
"count": 3136,
"observable_mse": 0.0003546668740455061,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04347893223166466
},
"reported": {
"count": 3136,
"observable_mse": 0.00035466704866848886,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04347893223166466
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016864181331919994
},
"comparisons": 7
},
{
"case": "seen_stays_in_support",
"status": "passed",
"actual": {
"count": 407,
"observable_mse": 0.00031717080855742097,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.040965866297483444
},
"reported": {
"count": 407,
"observable_mse": 0.00031717095407657325,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.040965866297483444
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014104559566900483
},
"comparisons": 7
},
{
"case": "held_initial",
"status": "passed",
"actual": {
"count": 448,
"observable_mse": 0.0003559494798537344,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04347892850637436
},
"reported": {
"count": 448,
"observable_mse": 0.0003559496544767171,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04347892850637436
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.7462298274040222e-10,
"numeric_tolerance_ratio": 0.00016862092668142262
},
"comparisons": 7
},
{
"case": "seen_enters_held",
"status": "passed",
"actual": {
"count": 2281,
"observable_mse": 0.00036110542714595795,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04391222447156906
},
"reported": {
"count": 2281,
"observable_mse": 0.00036110557266511023,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
},
"normalized_latent_rmse": 0.04391222447156906
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-10,
"numeric_tolerance_ratio": 0.00014044751427644966
},
"comparisons": 7
}
],
"saved_diagnostic_differences": []
}
],
"relations": [
{
"relation": "X_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006257921922951937
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5511306375847198e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5511330022709444e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-11,
"numeric_tolerance_ratio": 2.3586689464189724e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_X",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006255257874727249
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5392424504389055e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5392444513272494e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.000888343900442e-11,
"numeric_tolerance_ratio": 1.9958204678516046e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006481279619038105
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5999313947977498e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5999332137871534e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8142724164315183e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "inverse_R",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006476297043263912
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.6249348593410105e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.624933586048428e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2699590243860832e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "X_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.021607279777526855
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.766223668819293e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.7661858338397e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.7834979593753815e-10,
"numeric_tolerance_ratio": 0.00037580701583076647
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "Y_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.019717823714017868
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 6.143248901935294e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 6.143239443190396e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.40099235142008e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "S_order_7",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.020352382212877274
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 7.072931475704536e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 7.072944572428241e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.3096723705530167e-10,
"numeric_tolerance_ratio": 0.00013004741886984787
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "R_order_4",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.012590174563229084
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.74880874005612e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.748832386918366e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 2.3646862246096134e-10,
"numeric_tolerance_ratio": 0.000235585452088286
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "H_order_2",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 6.998432908560917e-16
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_inverse",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.005898219998925924
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.7603371563600376e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.760327151918318e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.000444171950221e-10,
"numeric_tolerance_ratio": 9.976902205452467e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.8386926715029404e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.8386933990987018e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 7.275957614183426e-12,
"numeric_tolerance_ratio": 7.2553618663453594e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_idempotent",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.0037216152995824814
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.5795047147548757e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.579504350956995e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.637978807091713e-12,
"numeric_tolerance_ratio": 3.6322416655772068e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4492263289866969e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4492271475319285e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 8.185452315956354e-12,
"numeric_tolerance_ratio": 8.17360690293936e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "rotation_conjugates_translation",
"group": "identities",
"status": "passed",
"metrics": {
"decoded_relation_agreement": 1.0,
"normalized_latent_relation_rmse": 0.006914579775184393
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.1585754186380655e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.158584877382964e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.458744898438454e-11,
"numeric_tolerance_ratio": 9.428962719383602e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5493387511232868e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.54933529504342e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 3.456079866737127e-11,
"numeric_tolerance_ratio": 3.447291564678985e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "translation_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5946865207515657e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.594685065560043e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.4551915228366852e-11,
"numeric_tolerance_ratio": 1.4514255306884354e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5181043383781798e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5181052478728816e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 9.094947017729282e-12,
"numeric_tolerance_ratio": 9.072102508792652e-06
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "context_flip_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.5509047191007994e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.5509052647976205e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 5.4569682106375694e-12,
"numeric_tolerance_ratio": 5.443083420483525e-06
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 2.650252827152144e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 2.650248643476516e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 4.18367562815547e-11,
"numeric_tolerance_ratio": 4.172617155200154e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_rotation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 96,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 2.8929631298524328e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 2.8929516702191904e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.1459633242338896e-10,
"numeric_tolerance_ratio": 0.00011426576708166119
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 96,
"observable_mse": 1.3614365343528334e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 96,
"observable_mse": 1.3614378076454159e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.2732925824820995e-11,
"numeric_tolerance_ratio": 1.2715614306756342e-05
},
"comparisons": 6
}
],
"differences": []
},
{
"relation": "projection_translation",
"group": "noncommuting",
"status": "passed",
"metrics": {
"oracle_distinguishing_states": 98,
"distinguished_fraction": 1.0
},
"comparisons": [
{
"case": "left",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 1.4222289792087395e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 1.4222278878150973e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.0913936421275139e-11,
"numeric_tolerance_ratio": 1.0898436361148734e-05
},
"comparisons": 6
},
{
"case": "right",
"status": "passed",
"actual": {
"count": 98,
"observable_mse": 3.5869325074600056e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"reported": {
"count": 98,
"observable_mse": 3.586930688470602e-05,
"exact_state_accuracy": 1.0,
"per_field_accuracy": {
"x": 1.0,
"y": 1.0,
"h": 1.0
}
},
"differences": [],
"max_errors": {
"accuracy_abs": 0.0,
"numeric_abs": 1.8189894035458565e-11,
"numeric_tolerance_ratio": 1.8124881342346815e-05
},
"comparisons": 6
}
],
"differences": []
}
],
"saved_h_count_groups": {
"length": 16,
"word_order": "frozen design order, zero-based",
"groups": [
{
"H_count": 0,
"word_indices": [
3,
16,
18,
30,
31
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 1,
"word_indices": [
0,
4,
8,
10,
13,
17,
19,
20,
28,
29
],
"count": 980,
"correct_count": 980,
"error_count": 0,
"per_field_correct_counts": {
"x": 980,
"y": 980,
"h": 980
}
},
{
"H_count": 2,
"word_indices": [
1,
2,
9,
11,
21,
22,
23
],
"count": 686,
"correct_count": 686,
"error_count": 0,
"per_field_correct_counts": {
"x": 686,
"y": 686,
"h": 686
}
},
{
"H_count": 3,
"word_indices": [
5,
6,
15,
25,
27
],
"count": 490,
"correct_count": 490,
"error_count": 0,
"per_field_correct_counts": {
"x": 490,
"y": 490,
"h": 490
}
},
{
"H_count": 4,
"word_indices": [
26
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 5,
"word_indices": [
7,
14
],
"count": 196,
"correct_count": 196,
"error_count": 0,
"per_field_correct_counts": {
"x": 196,
"y": 196,
"h": 196
}
},
{
"H_count": 7,
"word_indices": [
24
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
},
{
"H_count": 8,
"word_indices": [
12
],
"count": 98,
"correct_count": 98,
"error_count": 0,
"per_field_correct_counts": {
"x": 98,
"y": 98,
"h": 98
}
}
]
},
"h_count_group_differences": [],
"primary_length16": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"recorded_training": {
"mode": "official",
"H_edges": {
"count": 84,
"sha256": "78367f1012c1cf283ea902f25ce10450301a29a97ac65f4743b3fbde24d27c0d",
"total_training_edges": 580,
"objective_scale": 0.14482758620689656,
"scope": "H edges with both endpoints in registered training support"
},
"optimizer_updates": 0,
"optimizer_initial_state": {
"type": null,
"state": {}
},
"curve": [
{
"step": 0,
"losses": {
"prediction_mse": 2.3450114895240404e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 3.396223519303021e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 3.396223519303021e-06
},
"gradients": {
"norms": {
"prediction": 0.00011556209938134998,
"weighted_latent": 2.6831793128829187e-18,
"total": 0.00011556209938134998
},
"prediction_weighted_latent_dot": 1.8475759799887515e-23,
"prediction_weighted_latent_cosine": 0.05958503346418309,
"cosine_status": "defined",
"semantics": "Prediction and latent gradients include edge-frequency scale; latent includes weight 0.1"
},
"update_norm": 0.0
}
],
"checkpoints": [
{
"step": 0,
"path": "checkpoints/step-0000.json",
"sha256": "54d0f37cf21d2f3971d6eede53623bef7794973f5523f1f4b98bab3c2bb4a858"
}
],
"final_losses": {
"prediction_mse": 2.3450114895240404e-05,
"latent_mse": 3.151272073551623e-31,
"scaled_prediction": 3.396223519303021e-06,
"scaled_weighted_latent": 4.5639115510232414e-33,
"total": 3.396223519303021e-06
},
"training_seconds": 4.309997893869877e-05,
"evaluation_seconds": 0.6614362000254914
},
"recorded_training_scope": "Receipt-linked training logs for context; losses, gradients and Adam trajectory are not independently recomputed."
}
],
"verification": {
"status": "verified",
"mode": "official",
"trials": 20,
"optimizer_updates": 10000,
"source_sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573",
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"files_verified": 324,
"scope": "Pinned inputs, declared metadata migration, frozen tensors, optimizer records, outputs and resources; no retraining"
},
"receipt_sha256": "cf50bd54f71a55690e3aac7e28ca33cc63f07b750964c221e2e306dab0f73125",
"parent_receipt_sha256": "23470a50894fd6ec6bb3f1daefd2becf662a8722d2fa867c8bf43dd8dc13cfc1",
"run_source_sha256": "69fe95edc0f488df05e974528ddd5eba17bcc69934923e2c4c19d694f576c573",
"design_sha256": "25a3e7b8505d21d53b45d24c8a7f938f878e68d884fb1cbe6db9b900c8c902df",
"paired_correctness": [
{
"seed": 0,
"arm": "prediction_latent",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "prediction_latent",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "prediction_latent",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "prediction_latent",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "prediction_latent",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
},
"numpy": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
}
},
{
"seed": 0,
"arm": "latent_only",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "latent_only",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "latent_only",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "latent_only",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "latent_only",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
},
"numpy": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
}
},
{
"seed": 0,
"arm": "refit",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "refit",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "refit",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "refit",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 0,
"arm": "refit",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
},
"numpy": {
"count": 3136,
"correct_to_correct": 2955,
"correct_to_wrong": 0,
"wrong_to_correct": 181,
"wrong_to_wrong": 0,
"net_correct_gain": 181
}
},
{
"seed": 1,
"arm": "prediction_latent",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "prediction_latent",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "prediction_latent",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "prediction_latent",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
},
"numpy": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
}
},
{
"seed": 1,
"arm": "prediction_latent",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
},
"numpy": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
}
},
{
"seed": 1,
"arm": "latent_only",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "latent_only",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "latent_only",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "latent_only",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
},
"numpy": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
}
},
{
"seed": 1,
"arm": "latent_only",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
},
"numpy": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
}
},
{
"seed": 1,
"arm": "refit",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "refit",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "refit",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 1,
"arm": "refit",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
},
"numpy": {
"count": 3136,
"correct_to_correct": 3059,
"correct_to_wrong": 0,
"wrong_to_correct": 77,
"wrong_to_wrong": 0,
"net_correct_gain": 77
}
},
{
"seed": 1,
"arm": "refit",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
},
"numpy": {
"count": 3136,
"correct_to_correct": 3006,
"correct_to_wrong": 0,
"wrong_to_correct": 130,
"wrong_to_wrong": 0,
"net_correct_gain": 130
}
},
{
"seed": 2,
"arm": "prediction_latent",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "prediction_latent",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "prediction_latent",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "prediction_latent",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "prediction_latent",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
},
"numpy": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
}
},
{
"seed": 2,
"arm": "latent_only",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "latent_only",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "latent_only",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "latent_only",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "latent_only",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
},
"numpy": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
}
},
{
"seed": 2,
"arm": "refit",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "refit",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "refit",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "refit",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 2,
"arm": "refit",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
},
"numpy": {
"count": 3136,
"correct_to_correct": 2887,
"correct_to_wrong": 0,
"wrong_to_correct": 249,
"wrong_to_wrong": 0,
"net_correct_gain": 249
}
},
{
"seed": 3,
"arm": "prediction_latent",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 3,
"arm": "prediction_latent",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6061,
"correct_to_wrong": 47,
"wrong_to_correct": 67,
"wrong_to_wrong": 97,
"net_correct_gain": 20
},
"numpy": {
"count": 6272,
"correct_to_correct": 6061,
"correct_to_wrong": 47,
"wrong_to_correct": 67,
"wrong_to_wrong": 97,
"net_correct_gain": 20
}
},
{
"seed": 3,
"arm": "prediction_latent",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2808,
"correct_to_wrong": 64,
"wrong_to_correct": 123,
"wrong_to_wrong": 141,
"net_correct_gain": 59
},
"numpy": {
"count": 3136,
"correct_to_correct": 2808,
"correct_to_wrong": 64,
"wrong_to_correct": 123,
"wrong_to_wrong": 141,
"net_correct_gain": 59
}
},
{
"seed": 3,
"arm": "prediction_latent",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2097,
"correct_to_wrong": 165,
"wrong_to_correct": 398,
"wrong_to_wrong": 476,
"net_correct_gain": 233
},
"numpy": {
"count": 3136,
"correct_to_correct": 2097,
"correct_to_wrong": 165,
"wrong_to_correct": 398,
"wrong_to_wrong": 476,
"net_correct_gain": 233
}
},
{
"seed": 3,
"arm": "prediction_latent",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 1615,
"correct_to_wrong": 93,
"wrong_to_correct": 552,
"wrong_to_wrong": 876,
"net_correct_gain": 459
},
"numpy": {
"count": 3136,
"correct_to_correct": 1615,
"correct_to_wrong": 93,
"wrong_to_correct": 552,
"wrong_to_wrong": 876,
"net_correct_gain": 459
}
},
{
"seed": 3,
"arm": "latent_only",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 3,
"arm": "latent_only",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6108,
"correct_to_wrong": 0,
"wrong_to_correct": 164,
"wrong_to_wrong": 0,
"net_correct_gain": 164
},
"numpy": {
"count": 6272,
"correct_to_correct": 6108,
"correct_to_wrong": 0,
"wrong_to_correct": 164,
"wrong_to_wrong": 0,
"net_correct_gain": 164
}
},
{
"seed": 3,
"arm": "latent_only",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2872,
"correct_to_wrong": 0,
"wrong_to_correct": 264,
"wrong_to_wrong": 0,
"net_correct_gain": 264
},
"numpy": {
"count": 3136,
"correct_to_correct": 2872,
"correct_to_wrong": 0,
"wrong_to_correct": 264,
"wrong_to_wrong": 0,
"net_correct_gain": 264
}
},
{
"seed": 3,
"arm": "latent_only",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2262,
"correct_to_wrong": 0,
"wrong_to_correct": 874,
"wrong_to_wrong": 0,
"net_correct_gain": 874
},
"numpy": {
"count": 3136,
"correct_to_correct": 2262,
"correct_to_wrong": 0,
"wrong_to_correct": 874,
"wrong_to_wrong": 0,
"net_correct_gain": 874
}
},
{
"seed": 3,
"arm": "latent_only",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 1708,
"correct_to_wrong": 0,
"wrong_to_correct": 1428,
"wrong_to_wrong": 0,
"net_correct_gain": 1428
},
"numpy": {
"count": 3136,
"correct_to_correct": 1708,
"correct_to_wrong": 0,
"wrong_to_correct": 1428,
"wrong_to_wrong": 0,
"net_correct_gain": 1428
}
},
{
"seed": 3,
"arm": "refit",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 3,
"arm": "refit",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6108,
"correct_to_wrong": 0,
"wrong_to_correct": 164,
"wrong_to_wrong": 0,
"net_correct_gain": 164
},
"numpy": {
"count": 6272,
"correct_to_correct": 6108,
"correct_to_wrong": 0,
"wrong_to_correct": 164,
"wrong_to_wrong": 0,
"net_correct_gain": 164
}
},
{
"seed": 3,
"arm": "refit",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2872,
"correct_to_wrong": 0,
"wrong_to_correct": 264,
"wrong_to_wrong": 0,
"net_correct_gain": 264
},
"numpy": {
"count": 3136,
"correct_to_correct": 2872,
"correct_to_wrong": 0,
"wrong_to_correct": 264,
"wrong_to_wrong": 0,
"net_correct_gain": 264
}
},
{
"seed": 3,
"arm": "refit",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 2262,
"correct_to_wrong": 0,
"wrong_to_correct": 874,
"wrong_to_wrong": 0,
"net_correct_gain": 874
},
"numpy": {
"count": 3136,
"correct_to_correct": 2262,
"correct_to_wrong": 0,
"wrong_to_correct": 874,
"wrong_to_wrong": 0,
"net_correct_gain": 874
}
},
{
"seed": 3,
"arm": "refit",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 1708,
"correct_to_wrong": 0,
"wrong_to_correct": 1428,
"wrong_to_wrong": 0,
"net_correct_gain": 1428
},
"numpy": {
"count": 3136,
"correct_to_correct": 1708,
"correct_to_wrong": 0,
"wrong_to_correct": 1428,
"wrong_to_wrong": 0,
"net_correct_gain": 1428
}
},
{
"seed": 4,
"arm": "prediction_latent",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "prediction_latent",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "prediction_latent",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "prediction_latent",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "prediction_latent",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
},
"numpy": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
}
},
{
"seed": 4,
"arm": "latent_only",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "latent_only",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "latent_only",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "latent_only",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "latent_only",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
},
"numpy": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
}
},
{
"seed": 4,
"arm": "refit",
"baseline": "native",
"length": 1,
"status": "passed",
"saved": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 784,
"correct_to_correct": 784,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "refit",
"baseline": "native",
"length": 2,
"status": "passed",
"saved": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 6272,
"correct_to_correct": 6272,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "refit",
"baseline": "native",
"length": 4,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "refit",
"baseline": "native",
"length": 8,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
},
"numpy": {
"count": 3136,
"correct_to_correct": 3136,
"correct_to_wrong": 0,
"wrong_to_correct": 0,
"wrong_to_wrong": 0,
"net_correct_gain": 0
}
},
{
"seed": 4,
"arm": "refit",
"baseline": "native",
"length": 16,
"status": "passed",
"saved": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
},
"numpy": {
"count": 3136,
"correct_to_correct": 3062,
"correct_to_wrong": 0,
"wrong_to_correct": 74,
"wrong_to_wrong": 0,
"net_correct_gain": 74
}
}
],
"H_free_isolation": [
{
"seed": 0,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "latent_only",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 0,
"arm": "refit",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "latent_only",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 1,
"arm": "refit",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "latent_only",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 2,
"arm": "refit",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "latent_only",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 3,
"arm": "refit",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "prediction_latent",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "latent_only",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "reconstruction",
"backend": "saved",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "reconstruction",
"backend": "numpy",
"status": "passed",
"H_free_cases": 98,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 980,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 1568,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "one_step",
"backend": "saved",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "one_step",
"backend": "numpy",
"status": "passed",
"H_free_cases": 686,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 6860,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 10976,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_2",
"backend": "saved",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_2",
"backend": "numpy",
"status": "passed",
"H_free_cases": 4802,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 48020,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 76832,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_4",
"backend": "saved",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_4",
"backend": "numpy",
"status": "passed",
"H_free_cases": 1666,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 16660,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 26656,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_8",
"backend": "saved",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_8",
"backend": "numpy",
"status": "passed",
"H_free_cases": 784,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 12544,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_16",
"backend": "saved",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
},
{
"seed": 4,
"arm": "refit",
"set": "composition_16",
"backend": "numpy",
"status": "passed",
"H_free_cases": 490,
"arrays": {
"latent": {
"status": "passed",
"compared_scalars": 4900,
"differing_scalars": 0,
"witnesses": []
},
"logits": {
"status": "passed",
"compared_scalars": 7840,
"differing_scalars": 0,
"witnesses": []
}
}
}
],
"summary": {
"trials_audited": 20,
"trials_passed": 20,
"trials_failed": 0,
"distinct_saved_predictions": 331240,
"categorical_disagreements": 0,
"continuous_scalars_compared": 8631840,
"continuous_scalars_outside_tolerance": 0,
"continuous_witnesses_retained": 0,
"max_continuous_absolute_error": 7.748603820800781e-07,
"max_continuous_tolerance_ratio": 0.6967987208633615,
"aggregate_comparison_failures": 0,
"relation_failures": 0,
"paired_comparison_rows": 75,
"paired_comparison_failures": 0,
"H_free_isolation_rows": 180,
"H_free_isolation_failures": 0,
"primary_length16": [
{
"seed": 0,
"arm": "native",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 2955,
"accuracy": 0.9422831632653061
},
"numpy": {
"cases": 3136,
"correct": 2955,
"accuracy": 0.9422831632653061
}
},
"failure": null
},
{
"seed": 0,
"arm": "prediction_latent",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 0,
"arm": "latent_only",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 0,
"arm": "refit",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 1,
"arm": "prediction_latent",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 1,
"arm": "latent_only",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 1,
"arm": "refit",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 1,
"arm": "native",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3006,
"accuracy": 0.9585459183673469
},
"numpy": {
"cases": 3136,
"correct": 3006,
"accuracy": 0.9585459183673469
}
},
"failure": null
},
{
"seed": 2,
"arm": "latent_only",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 2,
"arm": "refit",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 2,
"arm": "native",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 2887,
"accuracy": 0.9205994897959183
},
"numpy": {
"cases": 3136,
"correct": 2887,
"accuracy": 0.9205994897959183
}
},
"failure": null
},
{
"seed": 2,
"arm": "prediction_latent",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 3,
"arm": "refit",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 3,
"arm": "native",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 1708,
"accuracy": 0.5446428571428571
},
"numpy": {
"cases": 3136,
"correct": 1708,
"accuracy": 0.5446428571428571
}
},
"failure": null
},
{
"seed": 3,
"arm": "prediction_latent",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 2167,
"accuracy": 0.6910076530612245
},
"numpy": {
"cases": 3136,
"correct": 2167,
"accuracy": 0.6910076530612245
}
},
"failure": null
},
{
"seed": 3,
"arm": "latent_only",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 4,
"arm": "native",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3062,
"accuracy": 0.9764030612244898
},
"numpy": {
"cases": 3136,
"correct": 3062,
"accuracy": 0.9764030612244898
}
},
"failure": null
},
{
"seed": 4,
"arm": "prediction_latent",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 4,
"arm": "latent_only",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
},
{
"seed": 4,
"arm": "refit",
"status": "passed",
"results": {
"saved": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
},
"numpy": {
"cases": 3136,
"correct": 3136,
"accuracy": 1.0
}
},
"failure": null
}
]
},
"ended_at": "2026-10-03T18:52:52.935961+00:00",
"elapsed_seconds": 13.982745499990415
}