H5json

README.md

@@ -1,5 +1,3 @@
-[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/hdfgroup/h5pyd)
-
 h5pyd
 =====
 

examples/swmr_multiprocess.py

@@ -36,6 +36,7 @@ def __init__(self, event, fname, dsetname, block_size, loop_count, sleep_time=0.
         self._event = event
         self._fname = fname
         self._dsetname = dsetname
+        self._block_size = block_size
         self._total_rows = block_size * loop_count
         self._sleep_time = sleep_time
         self._timeout = 5
@@ -45,6 +46,7 @@ def run(self):
         self.log.info("Waiting for initial event")
         assert self._event.wait(self._timeout)
         self._event.clear()
+        # time.sleep(5) works ok with the sleep
 
         self.log.info(f"Opening file {self._fname}")
         if self._fname.startswith("hdf5://"):
@@ -63,9 +65,9 @@ def run(self):
                 row_count = dset.shape[0]
                 if row_count > last_count:
                     self.log.info(f"Read {row_count - last_count} rows added")
-                    if row_count > last_count + block_size:
+                    if row_count > last_count + self._block_size:
                         # This selection should have data updated after a resize
-                        arr = dset[last_count:(last_count + block_size)]
+                        arr = dset[last_count:(last_count + self._block_size)]
                         self.log.info(f"Read data read, min: {arr.min()} max: {arr.max()}")
                     last_count = row_count
                 else:
@@ -99,8 +101,8 @@ def run(self):
 
         try:
             kwargs = {"dtype": np.dtype("int64"), "chunks": (1024 * 256,), "maxshape": (None,)}
-            if compression:
-                kwargs["compression"] = compression
+            if self._compression:
+                kwargs["compression"] = self._compression
             dset = f.create_dataset(self._dsetname, (0,), **kwargs)
             assert not f.swmr_mode
 

h5pyd/__init__.py

@@ -11,10 +11,10 @@
 ##############################################################################
 
 from __future__ import absolute_import
-
+from h5json.hdf5dtype import Reference, RegionReference
 from . import version
 from ._hl.base import Empty
-from ._hl.h5type import special_dtype, Reference, RegionReference
+from ._hl.h5type import special_dtype
 from ._hl.h5type import vlen_dtype, string_dtype, enum_dtype
 from ._hl.h5type import check_vlen_dtype, check_string_dtype, check_enum_dtype
 from ._hl.h5type import check_opaque_dtype, check_ref_dtype, check_dtype
@@ -25,7 +25,7 @@
 from ._hl.table import Table
 from ._hl.datatype import Datatype
 from ._hl.attrs import AttributeManager
-from ._hl.serverinfo import getServerInfo
+from .serverinfo import getServerInfo
 from . import h5ds
 
 

h5pyd/_apps/hsinfo.py

@@ -13,7 +13,10 @@
 import sys
 import logging
 import time
-import h5pyd
+
+from h5pyd.httpconn import HttpConn
+from h5pyd import Folder
+from h5pyd.version import version as h5pyd_version
 
 if __name__ == "__main__":
     from config import Config
@@ -47,8 +50,8 @@ def usage():
             print(f"    {help_msg}")
     print("")
     print("examples:")
-    print(f"   {cmd} -e http://hsdshdflab.hdfgroup.org")
-    print(f"   {cmd} -e http://hsdshdflab.hdfgroup.org /shared/tall.h5")
+    print(f"   {cmd} -e http://hsds.hdf.test")
+    print(f"   {cmd} -e http://hsds.hdf.test /shared/tall.h5")
     print(cfg.get_see_also(cmd))
     print("")
     sys.exit()
@@ -83,10 +86,19 @@ def getServerInfo(cfg):
     username = cfg["hs_username"]
     password = cfg["hs_password"]
     endpoint = cfg["hs_endpoint"]
+    http_conn = None
+
     try:
-        info = h5pyd.getServerInfo(
-            username=username, password=password, endpoint=endpoint
-        )
+        kwargs = {}
+        kwargs["username"] = username
+        kwargs["password"] = password
+        kwargs["endpoint"] = endpoint
+        http_conn = HttpConn(None, **kwargs)
+
+        http_conn.open()
+
+        info = http_conn.serverInfo()
+
         info_name = info["name"]
         print(f"server name: {info_name}")
         if "state" in info:
@@ -97,11 +109,9 @@ def getServerInfo(cfg):
             admin_tag = "(admin)"
         else:
             admin_tag = ""
-
         info_username = info["username"]
         print(f"username: {info_username} {admin_tag}")
-        info_password = info["password"]
-        print(f"password: {info_password}")
+
         if info["state"] == "READY":
             try:
                 home_folder = getHomeFolder()
@@ -119,7 +129,7 @@ def getServerInfo(cfg):
         if "start_time" in info:
             uptime = getUpTime(info["start_time"])
             print(f"up: {uptime}")
-        print(f"h5pyd version: {h5pyd.version.version}")
+        print(f"h5pyd version: {h5pyd_version}")
 
     except IOError as ioe:
         if ioe.errno == 401:
@@ -130,6 +140,9 @@ def getServerInfo(cfg):
                 print("authentication failure")
         else:
             print(f"Error: {ioe}")
+    finally:
+        if http_conn:
+            http_conn.close()
 
 
 #
@@ -141,9 +154,14 @@ def getHomeFolder():
     endpoint = cfg["hs_endpoint"]
     if not username:
         return None
-    dir = h5pyd.Folder(
-        "/home/", username=username, password=password, endpoint=endpoint
-    )  # get folder object for root
+
+    kwargs = {}
+    kwargs["username"] = username
+    kwargs["password"] = password
+    kwargs["endpoint"] = endpoint
+    dir = Folder("/home/", **kwargs)  # get folder object for root
+
+    # get folder object for root
     homefolder = None
     for name in dir:
         # we should come across the given domain
@@ -152,18 +170,18 @@ def getHomeFolder():
             # e.g. folder: "/home/bob/" for username "bob@acme.com"
             path = "/home/" + name + "/"
             try:
-                f = h5pyd.Folder(
-                    path, username=username, password=password, endpoint=endpoint
-                )
+                f = Folder(path, **kwargs)
+                if f.owner == username:
+                    homefolder = path
             except IOError as ioe:
                 logging.info(f"find home folder - got ioe: {ioe}")
                 continue
             except Exception as e:
                 logging.warning(f"find home folder - got exception: {e}")
                 continue
-            if f.owner == username:
-                homefolder = path
-            f.close()
+            finally:
+                if f:
+                    f.close()
             if homefolder:
                 break
 

h5pyd/_apps/utillib.py

@@ -162,7 +162,7 @@ def get_chunk_layout(dset):
         msg = "get_chunk_layout called on hdf5 dataset"
         logging.error(msg)
         raise IOError(msg)
-    dset_json = dset.id.dcpl_json
+    dset_json = dset.id.cpl_json
     if "layout" not in dset_json:
         msg = f"expect to find layout key in dset_json: {dset_json}"
         logging.error(msg)
@@ -1244,8 +1244,12 @@ def create_dataset(dobj, ctx):
             rank = 0
         else:
             tgt_shape.extend(dobj.shape)
-            tgt_maxshape.extend(dobj.maxshape)
             rank = len(tgt_shape)
+            if rank > 0:
+                tgt_maxshape.extend(dobj.maxshape)
+            else:
+                tgt_maxshape = None
+
         if rank > 0 and ctx["extend_dim"]:
             # set maxshape to unlimited for any dimension that is the extend_dim
             if dobj.name.split("/")[-1] == ctx["extend_dim"]:
@@ -1712,6 +1716,7 @@ def create_group(gobj, ctx):
             print(f"{gobj.name} not found")
 
         grp = fout.create_group(gobj.name)
+
         srcid_desobj_map = ctx["srcid_desobj_map"]
         msg = f"adding group id {gobj.id.id} to {grp} in srcid_desobj_map"
         logging.debug(msg)
@@ -1837,7 +1842,10 @@ def load_file(
 
     def copy_attribute_helper(name, obj):
         logging.info(f"copy attribute - name: {name}  obj: {obj.name}")
+        fout = ctx["fout"]
+
         tgt = fout[name]
+
         for a in obj.attrs:
             copy_attribute(tgt, a, obj, ctx)
 
@@ -1883,10 +1891,12 @@ def object_copy_helper(name, obj):
     # build a rough map of the file using the internal function above
     logging.info("creating target objects")
     fin.visititems(object_create_helper)
+    fout.flush()
 
     # copy over any attributes
     logging.info("creating target attributes")
     fin.visititems(copy_attribute_helper)
+    fout.flush()
 
     # create soft/external links (and hardlinks not already created)
     create_links(fin, fout, ctx)  # create root soft/external links