[WIP] Fixups

a3c.py

@@ -9,6 +9,16 @@
 
 import copy_param
 
+
+# Check if the hdf5 serializer that we want to use is avaliable
+try:
+    # will raise Runtime Error w/o h5py
+    serializers.save_hdf5(None,None)
+except AttributeError:
+    # because of None values
+    pass
+
+
 logger = getLogger(__name__)
 
 

a3c_ale.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import argparse
 import copy
 import multiprocessing as mp
@@ -27,12 +28,15 @@
 
 class A3CFF(chainer.ChainList, a3c.A3CModel):
 
-    def __init__(self, n_actions):
+    def __init__(self, n_actions, seed):
         self.head = dqn_head.NIPSDQNHead()
         self.pi = policy.FCSoftmaxPolicy(
-            self.head.n_output_channels, n_actions)
+            self.head.n_output_channels, n_actions, seed)
         self.v = v_function.FCVFunction(self.head.n_output_channels)
-        super().__init__(self.head, self.pi, self.v)
+        if sys.version_info < (3,0):
+            super(A3CFF, self).__init__(self.head, self.pi, self.v)
+        else:
+            super().__init__(self.head, self.pi, self.v)
         init_like_torch(self)
 
     def pi_and_v(self, state, keep_same_state=False):
@@ -42,14 +46,17 @@ def pi_and_v(self, state, keep_same_state=False):
 
 class A3CLSTM(chainer.ChainList, a3c.A3CModel):
 
-    def __init__(self, n_actions):
+    def __init__(self, n_actions, seed):
         self.head = dqn_head.NIPSDQNHead()
         self.pi = policy.FCSoftmaxPolicy(
-            self.head.n_output_channels, n_actions)
+            self.head.n_output_channels, n_actions, seed)
         self.v = v_function.FCVFunction(self.head.n_output_channels)
         self.lstm = L.LSTM(self.head.n_output_channels,
                            self.head.n_output_channels)
-        super().__init__(self.head, self.lstm, self.pi, self.v)
+        if sys.version_info < (3,0):
+            super(A3CLSTM, self).__init__(self.head, self.lstm, self.pi, self.v)
+        else:
+            super().__init__(self.head, self.lstm, self.pi, self.v)
         init_like_torch(self)
 
     def pi_and_v(self, state, keep_same_state=False):
@@ -207,20 +214,32 @@ def main():
     parser.set_defaults(use_lstm=False)
     args = parser.parse_args()
 
-    if args.seed is not None:
-        random_seed.set_random_seed(args.seed)
+    if args.seed is None:
+        args.seed = np.random.randint(0, 2 ** 16)
+
+
+    # I suggest using train_randstate instead of np.random because it proably
+    # behaves better for async use.
+    train_randstate = np.random.RandomState(args.seed)
+
+    # Choose random seed before async execution, in oder to assure
+    # that we obtain different seeds for each process. This can be checked
+    # by making sure each emulator has different seed, this works because each
+    # emulator is set to have the same random seeds as its process ( the ALE python
+    # class ) see ale.py for detials
+    process_seeds = train_randstate.randint(0, 2 ** 16, args.processes)
 
     args.outdir = prepare_output_dir(args, args.outdir)
 
     print('Output files are saved in {}'.format(args.outdir))
 
     n_actions = ale.ALE(args.rom).number_of_actions
 
-    def model_opt():
+    def model_opt(seed=args.seed):
         if args.use_lstm:
-            model = A3CLSTM(n_actions)
+            model = A3CLSTM(n_actions,seed=seed)
         else:
-            model = A3CFF(n_actions)
+            model = A3CFF(n_actions,seed=seed)
         opt = rmsprop_async.RMSpropAsync(lr=7e-4, eps=1e-1, alpha=0.99)
         opt.setup(model)
         opt.add_hook(chainer.optimizer.GradientClipping(40))
@@ -242,9 +261,15 @@ def model_opt():
         column_names = ('steps', 'elapsed', 'mean', 'median', 'stdev')
         print('\t'.join(column_names), file=f)
 
+    # convert np.int64 to python int for JSON
+    process_seeds = [int(x) for x in process_seeds]
+
     def run_func(process_idx):
-        env = ale.ALE(args.rom, use_sdl=args.use_sdl)
-        model, opt = model_opt()
+        env = ale.ALE(args.rom,
+                      seed=process_seeds[process_idx],
+                      use_sdl=args.use_sdl)
+
+        model, opt = model_opt(seed=process_seeds[process_idx])
         async.set_shared_params(model, shared_params)
         async.set_shared_states(opt, shared_states)
 

ale.py

@@ -4,9 +4,9 @@
 
 import numpy as np
 from ale_python_interface import ALEInterface
-import cv2
 
 import environment
+from utils import imresize
 
 
 class ALE(environment.EpisodicEnvironment):
@@ -27,9 +27,22 @@ def __init__(self, rom_filename, seed=None, use_sdl=False, n_last_screens=4,
             assert seed >= 0 and seed < 2 ** 16, \
                 "ALE's random seed must be represented by unsigned int"
         else:
-            # Use numpy's random state
+            # Warning Starting ALE without explicit random seeds can lead
+            # to all processes sharing the same inital state. Please check the
+            # args.txt in case you are concerned about this.
             seed = np.random.randint(0, 2 ** 16)
-        ale.setInt(b'random_seed', seed)
+
+        # Remember our (per process) random seed
+        self.seed = seed
+
+        # Intialize a random state for this thread. If we always call
+        # self.randstate instead of np.random it should make the process
+        # deterministic.
+        self.randstate = np.random.RandomState(self.seed)
+
+        # Use the random seed for the ALE too
+        ale.setInt(b'random_seed', self.seed)
+
         ale.setFloat(b'repeat_action_probability', 0.0)
         ale.setBool(b'color_averaging', False)
         if record_screen_dir is not None:
@@ -72,17 +85,15 @@ def current_screen(self):
         assert img.shape == (210, 160)
         if self.crop_or_scale == 'crop':
             # Shrink (210, 160) -> (110, 84)
-            img = cv2.resize(img, (84, 110),
-                             interpolation=cv2.INTER_LINEAR)
+            img = imresize(img, (84, 110))
             assert img.shape == (110, 84)
             # Crop (110, 84) -> (84, 84)
             unused_height = 110 - 84
             bottom_crop = 8
             top_crop = unused_height - bottom_crop
             img = img[top_crop: 110 - bottom_crop, :]
         elif self.crop_or_scale == 'scale':
-            img = cv2.resize(img, (84, 84),
-                             interpolation=cv2.INTER_LINEAR)
+            img = imresize(img, (84, 84))
         else:
             raise RuntimeError('crop_or_scale must be either crop or scale')
         assert img.shape == (84, 84)
@@ -144,7 +155,7 @@ def initialize(self):
             self.ale.reset_game()
 
         if self.max_start_nullops > 0:
-            n_nullops = np.random.randint(0, self.max_start_nullops + 1)
+            n_nullops = self.randstate.randint(0, self.max_start_nullops + 1)
             for _ in range(n_nullops):
                 self.ale.act(0)
 

async.py

@@ -75,6 +75,7 @@ def run_async(n_process, run_func):
 
     processes = []
 
+    # It is not clear to me that this does what it should. --max
     def set_seed_and_run(process_idx, run_func):
         random_seed.set_random_seed(np.random.randint(0, 2 ** 32))
         run_func(process_idx)

policy.py

@@ -4,6 +4,7 @@
 import chainer
 from chainer import functions as F
 from chainer import links as L
+import numpy as np
 
 import policy_output
 
@@ -26,19 +27,29 @@ def compute_logits(self, state):
         raise NotImplementedError
 
     def __call__(self, state):
-        return policy_output.SoftmaxPolicyOutput(self.compute_logits(state))
+        # The SoftmaxPolicyOutput is not persistent, so it cannot hold
+        # its own random state, rely instead on the policy randstate
+        # passed as a reference
+        return policy_output.SoftmaxPolicyOutput(
+            self.compute_logits(state),
+            self.policy_randstate)
 
 
 class FCSoftmaxPolicy(chainer.ChainList, SoftmaxPolicy):
     """Softmax policy that consists of FC layers and rectifiers"""
 
-    def __init__(self, n_input_channels, n_actions,
+    def __init__(self, n_input_channels, n_actions, seed,
                  n_hidden_layers=0, n_hidden_channels=None):
         self.n_input_channels = n_input_channels
         self.n_actions = n_actions
         self.n_hidden_layers = n_hidden_layers
         self.n_hidden_channels = n_hidden_channels
 
+        # Have a per policy randstate, this should provide diversity
+        # in the fact of similar environments
+        self.model_seed = seed
+        self.policy_randstate = np.random.RandomState(seed)
+
         layers = []
         if n_hidden_layers > 0:
             layers.append(L.Linear(n_input_channels, n_hidden_channels))

policy_output.py

@@ -9,7 +9,7 @@ class PolicyOutput(object):
     pass
 
 
-def _sample_discrete_actions(batch_probs):
+def _sample_discrete_actions(batch_probs, randstate):
     """Sample a batch of actions from a batch of action probabilities.
 
     Args:
@@ -31,8 +31,9 @@ def _sample_discrete_actions(batch_probs):
 
 class SoftmaxPolicyOutput(PolicyOutput):
 
-    def __init__(self, logits):
+    def __init__(self, logits, randstate):
         self.logits = logits
+        self.policy_output_randstate = randstate
 
     @cached_property
     def most_probable_actions(self):
@@ -48,7 +49,7 @@ def log_probs(self):
 
     @cached_property
     def action_indices(self):
-        return _sample_discrete_actions(self.probs.data)
+        return _sample_discrete_actions(self.probs.data, self.policy_output_randstate)
 
     @cached_property
     def sampled_actions_log_probs(self):

prepare_output_dir.py

@@ -3,6 +3,11 @@
 import json
 import subprocess
 
+import sys
+if sys.version_info < (3,0):
+    def tmp(cmd):
+        return subprocess.check_output(cmd, shell=True)
+    subprocess.getoutput = tmp
 
 def prepare_output_dir(args, user_specified_dir=None):
     """Prepare output directory.
@@ -31,7 +36,7 @@ def prepare_output_dir(args, user_specified_dir=None):
 
     # Save all the arguments
     with open(os.path.join(outdir, 'args.txt'), 'w') as f:
-        f.write(json.dumps(vars(args)))
+        f.write(json.dumps(vars(args))+"\n\n")
 
     # Save `git status`
     with open(os.path.join(outdir, 'git-status.txt'), 'w') as f:

run_a3c.py

@@ -9,7 +9,6 @@
 import chainer
 from chainer import links as L
 from chainer import functions as F
-import cv2
 import numpy as np
 
 import a3c

train_a3c_doom.py

@@ -4,7 +4,6 @@
 import chainer
 from chainer import links as L
 from chainer import functions as F
-import cv2
 import numpy as np
 
 import policy
@@ -16,10 +15,11 @@
 from init_like_torch import init_like_torch
 import run_a3c
 import doom_env
+from utils import imresize
 
 
 def phi(obs):
-    resized = cv2.resize(obs.image_buffer, (84, 84))
+    resized = imresize(obs.image_buffer, (84, 84))
     return resized.transpose(2, 0, 1).astype(np.float32) / 255
 
 

utils.py

@@ -0,0 +1,19 @@
+# There are a number of options for doing this:
+# 1. opencv
+# 2. PIL resize
+# 3. scipy interpn
+# 4. scipy.misc.imresize ( bad option as this in turn calls PIL )
+#
+# Does anyone know which is the fastest?
+
+try:
+   import cv2
+   imresize = cv2.resize
+except:
+   import numpy as np
+   import PIL.Image
+
+   def tmp(img,size):
+        return np.array(PIL.Image.fromarray(img).resize(size,PIL.Image.BILINEAR))
+
+   imresize = tmp