Toy1D Exp1 - Fix scale

Author: EltonCN

dev/toy1d_experiments/fix_results.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 55,
    "id": "09c54280",
    "metadata": {},
    "outputs": [],
@@ -18,17 +18,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 56,
    "id": "d58ad45e",
    "metadata": {},
    "outputs": [],
    "source": [
-    "paths = glob.glob(\"toy1d_experiment0_protocol_test\\\\*\\\\run*\")"
+    "EXPERIMENT0 = \"toy1d_experiment0_protocol_test\\\\*\\\\run*\"\n",
+    "EXPERIMENT1 = \"toy1d_experiment1_configuration_test\\\\*\\\\run*\"\n",
+    "EXPERIMENT2 = \"toy1d_experiment2_best_long\\\\*\\\\run*\"\n",
+    "\n",
+    "path_template = EXPERIMENT1\n",
+    "\n",
+    "paths = glob.glob(path_template)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 57,
    "id": "72cf638e",
    "metadata": {},
    "outputs": [],
@@ -63,7 +69,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 58,
    "id": "d35f852b",
    "metadata": {},
    "outputs": [],
@@ -75,121 +81,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "id": "6154870b",
    "metadata": {},
    "outputs": [],
    "source": [
     "for run_path in paths:\n",
     "    for name in patch_map:\n",
+    "        if name != \"metrics\" and path_template != EXPERIMENT0:\n",
+    "            continue\n",
+    "\n",
     "        metrics = load_metrics(run_path, name)\n",
     "        metrics = patch_map[name](metrics, run_path)\n",
     "        save_metrics(metrics, run_path, name)\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "id": "e702b2f7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import time\n",
-    "from typing import Any\n",
-    "\n",
-    "import torch\n",
-    "import tqdm\n",
-    "from hamilton import driver\n",
-    "from hamilton.function_modifiers import source, value\n",
-    "from numba import cuda\n",
-    "\n",
-    "from world_machine.data import WorldMachineDataset\n",
-    "from world_machine_experiments import shared\n",
-    "from world_machine_experiments.shared import function_variation\n",
-    "from world_machine_experiments.shared.load_multiple_metrics import (\n",
-    "    load_multiple_metrics)\n",
-    "from world_machine_experiments.shared.pipeline import save_pipeline\n",
-    "from world_machine_experiments.shared.save_metrics import (\n",
-    "    load_metrics, save_metrics)\n",
-    "from world_machine_experiments.shared.save_parameters import save_parameters\n",
-    "from world_machine_experiments.shared.save_plots import save_plots\n",
-    "from world_machine_experiments.shared.statistics import consolidated_metrics\n",
-    "from world_machine_experiments.shared.train_plots import train_plots\n",
-    "from world_machine_experiments.toy1d import base\n",
-    "from world_machine_experiments.toy1d.base import toy1d_masks_sensory_plots\n",
-    "\n",
-    "aditional_outputs = [\"save_toy1d_metrics\",\n",
-    "                         \"save_toy1d_metrics_sample_logits\",\n",
-    "                         \"save_toy1d_metrics_sample_plots\",\n",
-    "\n",
-    "                         \"save_toy1d_mask_sensory_plot\",\n",
-    "                         \"save_toy1d_mask_sensory_metrics\",\n",
-    "\n",
-    "                         \"save_toy1d_autoregressive_metrics\"]\n",
-    "\n",
-    "\n",
-    "d = driver.Builder().with_modules(base, shared).build()\n",
-    "\n",
-    "final_vars = [\"toy1d_train_history\",\n",
-    "                          \"save_toy1d_model\",\n",
-    "                          \"save_toy1d_train_history\",\n",
-    "                          \"save_toy1d_train_plots\",\n",
-    "                          \"save_toy1d_prediction_plots\",\n",
-    "                          \"toy1d_datasets\"]\n",
-    "\n",
-    "final_vars += aditional_outputs\n",
-    "\n",
-    "save_pipeline(d, final_vars, \"pipeline\", \".\", True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "id": "836a3f8e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from world_machine_experiments.shared.pipeline import save_pipeline, load_graph"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "id": "8a7f048a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "graph = load_graph(\"pipeline\", \".\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "id": "b0e08812",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "grafo2 = otimizar_layout_grafo_td_corrigido(graph)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 36,
-   "id": "72573ffc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "grafo2.write_svg(\"pipeline2.svg\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "abfdb6b4",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

experiments/src/world_machine_experiments/toy1d/specific/experiment1/joint_impact_test.py

@@ -144,20 +144,20 @@ def disjoint_mask(disjoint_groups: list[set[str]]) -> np.ndarray:
     return result
 
 
+EXPONENTS = {"normal": 4,
+             "use_state": 4,
+             "prediction": 2,
+             "prediction_shallow": 2,
+             "prediction_local": 3}
+
+
 def joint_impact_plots(joint_impact: dict,
                        disjoint_mask: np.ndarray) -> dict[str, Figure]:
 
     figures = {}
     for task in task_names:
-        if task in ["prediction_shallow", "prediction"]:
-            scale = 1e3
-            scale_exponent = 3
-        elif task in ["normal"]:
-            scale = 1e5
-            scale_exponent = 5
-        else:
-            scale = 1e4
-            scale_exponent = 4
+        scale_exponent = EXPONENTS[task]
+        scale = 10**scale_exponent
 
         fig = plt.figure(dpi=600)
 

experiments/src/world_machine_experiments/toy1d/specific/experiment1/prepare_data.py

@@ -32,7 +32,8 @@ def _add_diverge_mask(df: pd.DataFrame) -> None:
         data = df[f"{metric}_mse"].to_numpy()
 
         data_mask = np.bitwise_not(np.isnan(data))
-        data_mask = np.bitwise_and(data_mask, data < 1)
+        data_mask = np.bitwise_and(data_mask, np.bitwise_not(np.isinf(data)))
+        data_mask = np.bitwise_and(data_mask, data < 1000)
 
         data_max = data[data_mask].mean() + 3*data[data_mask].std()