From e3c3e4b2339959dc8c66b98d23ae79127150702b Mon Sep 17 00:00:00 2001 From: VarenyaJ Date: Tue, 28 Jul 2026 12:05:06 -0400 Subject: [PATCH] add fenominal gap demonstrations to walkthrough --- walkthrough.ipynb | 9586 +++++++++------------------------------------ 1 file changed, 1805 insertions(+), 7781 deletions(-) diff --git a/walkthrough.ipynb b/walkthrough.ipynb index d1c64f1..469ae57 100644 --- a/walkthrough.ipynb +++ b/walkthrough.ipynb @@ -32,10 +32,10 @@ "id": "65c5f47a", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.546963Z", - "iopub.status.busy": "2026-07-27T16:39:42.546803Z", - "iopub.status.idle": "2026-07-27T16:39:42.554516Z", - "shell.execute_reply": "2026-07-27T16:39:42.553785Z" + "iopub.execute_input": "2026-07-28T13:28:13.977102Z", + "iopub.status.busy": "2026-07-28T13:28:13.976939Z", + "iopub.status.idle": "2026-07-28T13:28:13.987879Z", + "shell.execute_reply": "2026-07-28T13:28:13.987179Z" } }, "outputs": [ @@ -86,10 +86,10 @@ "id": "193726a3", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.556269Z", - "iopub.status.busy": "2026-07-27T16:39:42.556158Z", - "iopub.status.idle": "2026-07-27T16:39:42.558656Z", - "shell.execute_reply": "2026-07-27T16:39:42.558057Z" + "iopub.execute_input": "2026-07-28T13:28:13.990232Z", + "iopub.status.busy": "2026-07-28T13:28:13.990095Z", + "iopub.status.idle": "2026-07-28T13:28:13.993081Z", + "shell.execute_reply": "2026-07-28T13:28:13.992273Z" } }, "outputs": [ @@ -124,10 +124,10 @@ "id": "70816661", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.560082Z", - "iopub.status.busy": "2026-07-27T16:39:42.559987Z", - "iopub.status.idle": "2026-07-27T16:39:42.787664Z", - "shell.execute_reply": "2026-07-27T16:39:42.787363Z" + "iopub.execute_input": "2026-07-28T13:28:13.995471Z", + "iopub.status.busy": "2026-07-28T13:28:13.995295Z", + "iopub.status.idle": "2026-07-28T13:28:14.084811Z", + "shell.execute_reply": "2026-07-28T13:28:14.084435Z" } }, "outputs": [ @@ -162,10 +162,10 @@ "id": "3aa1d6f3", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.788650Z", - "iopub.status.busy": "2026-07-27T16:39:42.788584Z", - "iopub.status.idle": "2026-07-27T16:39:42.793020Z", - "shell.execute_reply": "2026-07-27T16:39:42.792785Z" + "iopub.execute_input": "2026-07-28T13:28:14.085835Z", + "iopub.status.busy": "2026-07-28T13:28:14.085756Z", + "iopub.status.idle": "2026-07-28T13:28:14.088778Z", + "shell.execute_reply": "2026-07-28T13:28:14.088417Z" } }, "outputs": [ @@ -206,10 +206,10 @@ "id": "64921e1c", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.793978Z", - "iopub.status.busy": "2026-07-27T16:39:42.793937Z", - "iopub.status.idle": "2026-07-27T16:39:42.849179Z", - "shell.execute_reply": "2026-07-27T16:39:42.848907Z" + "iopub.execute_input": "2026-07-28T13:28:14.089628Z", + "iopub.status.busy": "2026-07-28T13:28:14.089584Z", + "iopub.status.idle": "2026-07-28T13:28:14.114466Z", + "shell.execute_reply": "2026-07-28T13:28:14.114170Z" } }, "outputs": [ @@ -217,90 +217,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" ] }, @@ -338,10 +266,10 @@ "id": "5cb532d8", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.850158Z", - "iopub.status.busy": "2026-07-27T16:39:42.850112Z", - "iopub.status.idle": "2026-07-27T16:39:42.851549Z", - "shell.execute_reply": "2026-07-27T16:39:42.851290Z" + "iopub.execute_input": "2026-07-28T13:28:14.115588Z", + "iopub.status.busy": "2026-07-28T13:28:14.115546Z", + "iopub.status.idle": "2026-07-28T13:28:14.117049Z", + "shell.execute_reply": "2026-07-28T13:28:14.116740Z" } }, "outputs": [ @@ -375,10 +303,10 @@ "id": "b8011af3", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.852485Z", - "iopub.status.busy": "2026-07-27T16:39:42.852438Z", - "iopub.status.idle": "2026-07-27T16:39:42.865866Z", - "shell.execute_reply": "2026-07-27T16:39:42.865496Z" + "iopub.execute_input": "2026-07-28T13:28:14.117834Z", + "iopub.status.busy": "2026-07-28T13:28:14.117796Z", + "iopub.status.idle": "2026-07-28T13:28:14.130769Z", + "shell.execute_reply": "2026-07-28T13:28:14.130434Z" } }, "outputs": [ @@ -386,111 +314,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250mm, percentile=42.5%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250mm, percentile=42.5%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" ] }, @@ -539,16 +377,26 @@ "`HpoConceptRecognizer` is an abstract base class (`abc.ABCMeta` + `@abc.abstractmethod`) - the real extension point for any text-to-HPO backend. `FenominalConceptRecognizer` is its one concrete implementation today, wrapping the third-party `fenominal` library. Every hit comes back as a `SimpleTerm` (`hpo/simple_term.py`) - `hpo_id`, `hpo_label`, `excluded`, `gestational_age`. Tested in `test_fenominal_cr.py`, `test_hpo_parser.py`." ] }, + { + "cell_type": "markdown", + "id": "3b335023", + "metadata": {}, + "source": [ + "### What fenominal actually is\n", + "\n", + "This is the tool doing the real work below, so it's worth naming plainly before using it: fenominal reads a doctor's free-text note and finds the HPO terms hiding inside it. Give it a sentence like \"no evidence of macrocephaly\" and it finds \"macrocephaly,\" looks up its HPO code, and also notices the word \"no\" so it can mark that finding as absent instead of present. `HpoParser` (used right below) is just a thin loader around it - fenominal is the part that actually reads the text." + ] + }, { "cell_type": "code", "execution_count": 8, "id": "241744aa", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:42.867898Z", - "iopub.status.busy": "2026-07-27T16:39:42.867859Z", - "iopub.status.idle": "2026-07-27T16:39:45.067378Z", - "shell.execute_reply": "2026-07-27T16:39:45.066968Z" + "iopub.execute_input": "2026-07-28T13:28:14.133234Z", + "iopub.status.busy": "2026-07-28T13:28:14.133195Z", + "iopub.status.idle": "2026-07-28T13:28:16.907762Z", + "shell.execute_reply": "2026-07-28T13:28:16.907344Z" } }, "outputs": [ @@ -556,220 +404,143 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:hpotk.store._github:Pulling tag from https://api.github.com/repos/obophenotype/human-phenotype-ontology/tags\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "DEBUG:hpotk.store._github:Pulling tag from https://api.github.com/repos/obophenotype/human-phenotype-ontology/tags\n", + "DEBUG:hpotk.store._github:Fetched 30 tags\n", + "DEBUG:hpotk.util:Using default encoding 'utf-8'\n", + "DEBUG:hpotk.util:Opening /Users/jv2684/.hpo-toolkit/HP/hp.v2026-06-23.json\n", + "DEBUG:hpotk.util:Looks like a local file: /Users/jv2684/.hpo-toolkit/HP/hp.v2026-06-23.json\n", + "DEBUG:hpotk.util:Looks like decompressed data\n", + "DEBUG:hpotk.ontology.load.obographs._load:Extracting ontology terms\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n", + "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/BFO_0000051', 'lbl': 'has part', 'meta': {'xrefs': [{'val': 'BFO:0000051'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'has_part'}]}}\n", + "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/BFO_0000066', 'lbl': 'occurs in', 'meta': {'xrefs': [{'val': 'BFO:0000066'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'occurs_in'}]}}\n", + "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/RO_0002503', 'lbl': 'towards', 'meta': {'xrefs': [{'val': 'RO:0002503'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'towards'}]}}\n", + "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/RO_0002573', 'lbl': 'has modifier', 'meta': {'comments': ['placeholder relation to indicate normality/abnormality.'], 'xrefs': [{'val': 'RO:0002180'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'qualifier'}]}}\n", + "DEBUG:hpotk.ontology.load.obographs._load:Creating the edge list\n", + "DEBUG:hpotk.ontology.load.obographs._load:Building ontology graph\n", + "DEBUG:hpotk.graph._factory:Creating ontology graph from 24378 edges\n", + "DEBUG:hpotk.graph._factory:Found root HP:0000001\n", + "DEBUG:hpotk.graph._factory:Extracted 19836 nodes\n", + "DEBUG:hpotk.ontology.load.obographs._load:Assembling the ontology\n", + "DEBUG:hpotk.ontology.load.obographs._load:Done\n", + "DEBUG:hpotk.store._github:Pulling tag from https://api.github.com/repos/obophenotype/human-phenotype-ontology/tags\n", "DEBUG:hpotk.store._github:Fetched 30 tags\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.util:Using default encoding 'utf-8'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.util:Opening /Users/jv2684/.hpo-toolkit/HP/hp.v2026-06-23.json\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.util:Looks like a local file: /Users/jv2684/.hpo-toolkit/HP/hp.v2026-06-23.json\n" - ] - }, - { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:hpotk.util:Looks like decompressed data\n" + "FenominalConceptRecognizer is-a HpoConceptRecognizer\n", + "HPO version: 2026-06-23\n" ] - }, + } + ], + "source": [ + "from prenatalppkt.hpo import HpoParser\n", + "\n", + "hpo_parser = HpoParser()\n", + "hpo_cr = hpo_parser.get_hpo_concept_recognizer()\n", + "print(type(hpo_cr).__name__, \"is-a\", type(hpo_cr).__mro__[1].__name__)\n", + "print(\"HPO version:\", hpo_parser.get_version())" + ] + }, + { + "cell_type": "markdown", + "id": "1db06636", + "metadata": {}, + "source": [ + "### Running it on Apple Sally's real clinical text" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "60ca5bb3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.908836Z", + "iopub.status.busy": "2026-07-28T13:28:16.908787Z", + "iopub.status.idle": "2026-07-28T13:28:16.910259Z", + "shell.execute_reply": "2026-07-28T13:28:16.909977Z" + } + }, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:hpotk.ontology.load.obographs._load:Extracting ontology terms\n" + "The sonogram was significant for splaying of the cerebellar hemispheres. There was evidence of a cyst measuring __ connecting with the fourth ventricle. This is consistent with ...\n" ] - }, + } + ], + "source": [ + "impression_text = raw[\"finalize\"][\"generalComment\"][\"plain_text\"]\n", + "print(impression_text[:400], \"...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "2558194f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.911029Z", + "iopub.status.busy": "2026-07-28T13:28:16.910989Z", + "iopub.status.idle": "2026-07-28T13:28:16.916088Z", + "shell.execute_reply": "2026-07-28T13:28:16.915768Z" + } + }, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._factory:Unknown synonym type http://purl.obolibrary.org/obo/hp#allelic_requirement\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/BFO_0000051', 'lbl': 'has part', 'meta': {'xrefs': [{'val': 'BFO:0000051'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'has_part'}]}}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/BFO_0000066', 'lbl': 'occurs in', 'meta': {'xrefs': [{'val': 'BFO:0000066'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'occurs_in'}]}}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/RO_0002503', 'lbl': 'towards', 'meta': {'xrefs': [{'val': 'RO:0002503'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'towards'}]}}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.io.obographs:Missing node type in {'id': 'http://purl.obolibrary.org/obo/RO_0002573', 'lbl': 'has modifier', 'meta': {'comments': ['placeholder relation to indicate normality/abnormality.'], 'xrefs': [{'val': 'RO:0002180'}], 'basicPropertyValues': [{'pred': 'http://www.geneontology.org/formats/oboInOwl#shorthand', 'val': 'qualifier'}]}}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._load:Creating the edge list\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._load:Building ontology graph\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.graph._factory:Creating ontology graph from 24378 edges\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.graph._factory:Found root HP:0000001\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.graph._factory:Extracted 19836 nodes\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._load:Assembling the ontology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.ontology.load.obographs._load:Done\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.store._github:Pulling tag from https://api.github.com/repos/obophenotype/human-phenotype-ontology/tags\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:hpotk.store._github:Fetched 30 tags\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "FenominalConceptRecognizer is-a HpoConceptRecognizer\n", - "HPO version: 2026-06-23\n" + "HP:0000256 Macrocephaly (excluded=True)\n", + "HP:0002119 Ventriculomegaly (excluded=True)\n", + "HP:0001274 Agenesis of corpus callosum (excluded=False)\n" ] } ], "source": [ - "from prenatalppkt.hpo import HpoParser\n", - "\n", - "hpo_parser = HpoParser()\n", - "hpo_cr = hpo_parser.get_hpo_concept_recognizer()\n", - "print(type(hpo_cr).__name__, \"is-a\", type(hpo_cr).__mro__[1].__name__)\n", - "print(\"HPO version:\", hpo_parser.get_version())" + "hits = hpo_cr.parse(impression_text)\n", + "for hit in hits:\n", + " print(f\"{hit.hpo_id} {hit.hpo_label} (excluded={hit.excluded})\")" ] }, { "cell_type": "markdown", - "id": "1db06636", + "id": "7a913b54", "metadata": {}, "source": [ - "### Running it on Apple Sally's real clinical text" + "**Worth knowing:** the \"no evidence of macrocephaly, ventriculomegaly or agenesis of the corpus callosum\" sentence isn't organic sonographer documentation - it was added by hand to Apple Sally's case specifically as a simple, deliberate negation-extraction test for the text-mining tool. At the time it was written, that tool was `FastHPOCR`; `fenominal` picks up the first two terms in that list correctly today too. The third term is where it gets interesting - see the next cell." + ] + }, + { + "cell_type": "markdown", + "id": "79f8fa84", + "metadata": {}, + "source": [ + "### A negation-scope inconsistency this session found\n", + "\n", + "Apple Sally's sentence negates three findings in one clause: \"no evidence of macrocephaly, ventriculomegaly **or** agenesis of the corpus callosum.\" Running the full impression text through `hpo_cr.parse()` two cells up, the first two terms came back `excluded=True` - correctly negated - but agenesis of the corpus callosum came back `excluded=False`, as if it were a positive finding, despite sitting in the exact same negated list. Isolating just that clause reproduces it directly:" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "60ca5bb3", + "execution_count": 11, + "id": "a0111073", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.068442Z", - "iopub.status.busy": "2026-07-27T16:39:45.068395Z", - "iopub.status.idle": "2026-07-27T16:39:45.069878Z", - "shell.execute_reply": "2026-07-27T16:39:45.069643Z" + "iopub.execute_input": "2026-07-28T13:28:16.917000Z", + "iopub.status.busy": "2026-07-28T13:28:16.916957Z", + "iopub.status.idle": "2026-07-28T13:28:16.918443Z", + "shell.execute_reply": "2026-07-28T13:28:16.918208Z" } }, "outputs": [ @@ -777,29 +548,39 @@ "name": "stdout", "output_type": "stream", "text": [ - "The patient was referred for a fetal anatomical survey. \r", - "\r", - "Sonographic measurements were consistent with the expected gestational age. The amniotic fluid volume was normal. A detailed fetal anatomic survey was performed. \r", - "\r", - "The sonogram was significant for splaying of the cerebellar hemispheres. There was evidence of a cyst measuring __ connecting with the fourth ventricle. This is consistent with ...\n" + "HP:0000256 Macrocephaly excluded=True\n", + "HP:0002119 Ventriculomegaly excluded=True\n", + "HP:0001274 Agenesis of corpus callosum excluded=False\n" ] } ], "source": [ - "impression_text = raw[\"finalize\"][\"generalComment\"][\"plain_text\"]\n", - "print(impression_text[:400], \"...\")" + "negation_clause = (\n", + " \"There was no evidence of macrocephaly, ventriculomegaly or \"\n", + " \"agenesis of the corpus callosum.\"\n", + ")\n", + "for hit in hpo_cr.parse(negation_clause):\n", + " print(f\"{hit.hpo_id} {hit.hpo_label} excluded={hit.excluded}\")" + ] + }, + { + "cell_type": "markdown", + "id": "6929976c", + "metadata": {}, + "source": [ + "**The gap this session found and documented:** Apple Sally's text says, word for word, \"This is consistent with a Dandy-Walker malformation\" - the exact official HPO label for `HP:0001305`. It's not in the list above. And unlike some of the other gaps found (where the exact phrase works fine on its own, just not inside a full sentence), this one is a true miss in every phrasing tried - not a sentence-context problem. Tracked as an honest `xfail` test (`test_dandy_walker_malformation_recognized_in_anatomy_text` in `test_fetal_anatomy.py`, plus a builder-level companion in `test_observer_phenopacket.py`) rather than hidden behind a loose assertion." ] }, { "cell_type": "code", - "execution_count": 10, - "id": "2558194f", + "execution_count": 12, + "id": "35d938d7", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.070612Z", - "iopub.status.busy": "2026-07-27T16:39:45.070570Z", - "iopub.status.idle": "2026-07-27T16:39:45.074358Z", - "shell.execute_reply": "2026-07-27T16:39:45.074061Z" + "iopub.execute_input": "2026-07-28T13:28:16.919270Z", + "iopub.status.busy": "2026-07-28T13:28:16.919226Z", + "iopub.status.idle": "2026-07-28T13:28:16.920787Z", + "shell.execute_reply": "2026-07-28T13:28:16.920475Z" } }, "outputs": [ @@ -807,44 +588,38 @@ "name": "stdout", "output_type": "stream", "text": [ - "HP:0000256 Macrocephaly (excluded=True)\n", - "HP:0002119 Ventriculomegaly (excluded=True)\n", - "HP:0001274 Agenesis of corpus callosum (excluded=False)\n" + "'Dandy-Walker malformation' -> []\n", + "'Dandy Walker malformation' -> []\n", + "'Dandy-Walker syndrome' -> []\n" ] } ], "source": [ - "hits = hpo_cr.parse(impression_text)\n", - "for hit in hits:\n", - " print(f\"{hit.hpo_id} {hit.hpo_label} (excluded={hit.excluded})\")" - ] - }, - { - "cell_type": "markdown", - "id": "7a913b54", - "metadata": {}, - "source": [ - "**Worth knowing:** the \"no evidence of macrocephaly, ventriculomegaly or agenesis of the corpus callosum\" sentence isn't organic sonographer documentation - it was added by hand to Apple Sally's case specifically as a simple, deliberate negation-extraction test for the text-mining tool. At the time it was written, that tool was `FastHPOCR`; `fenominal` picks it up correctly today too (mostly - see the note below on `excluded` below), which is a reasonable sign the basic case still holds across the tool swap." + "for phrasing in [\"Dandy-Walker malformation\", \"Dandy Walker malformation\", \"Dandy-Walker syndrome\"]:\n", + " hits = hpo_cr.parse(phrasing)\n", + " print(f\"{phrasing!r:32s} -> {[(h.hpo_id, h.hpo_label) for h in hits]}\")" ] }, { "cell_type": "markdown", - "id": "6929976c", + "id": "4cc0c141", "metadata": {}, "source": [ - "**The gap this session found and documented:** Apple Sally's text says, word for word, \"This is consistent with a Dandy-Walker malformation\" - the exact official HPO label for `HP:0001305`. It's not in the list above. And unlike some of the other gaps found (where the exact phrase works fine on its own, just not inside a full sentence), this one is a true miss in every phrasing tried - not a sentence-context problem. Tracked as an honest `xfail` test (`test_dandy_walker_malformation_recognized_in_anatomy_text` in `test_fetal_anatomy.py`, plus a builder-level companion in `test_observer_phenopacket.py`) rather than hidden behind a loose assertion." + "### fenominal across all six real clinical-text fixtures\n", + "\n", + "Beyond Apple Sally, every other fixture in `tests/data/` has its own real, independently-written clinical impression text. Running the same `hpo_cr.parse()` across all of them gives a broader read on where fenominal actually holds up, rather than judging it from the one case that was specifically built to test it:" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "35d938d7", + "execution_count": 13, + "id": "b7bccdbc", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.075190Z", - "iopub.status.busy": "2026-07-27T16:39:45.075151Z", - "iopub.status.idle": "2026-07-27T16:39:45.076691Z", - "shell.execute_reply": "2026-07-27T16:39:45.076389Z" + "iopub.execute_input": "2026-07-28T13:28:16.921622Z", + "iopub.status.busy": "2026-07-28T13:28:16.921571Z", + "iopub.status.idle": "2026-07-28T13:28:16.929792Z", + "shell.execute_reply": "2026-07-28T13:28:16.929530Z" } }, "outputs": [ @@ -852,16 +627,55 @@ "name": "stdout", "output_type": "stream", "text": [ - "'Dandy-Walker malformation' -> []\n", - "'Dandy Walker malformation' -> []\n", - "'Dandy-Walker syndrome' -> []\n" + "=== Apple_Sally_pretty.json ===\n", + " HP:0000256 Macrocephaly excluded=True\n", + " HP:0002119 Ventriculomegaly excluded=True\n", + " HP:0001274 Agenesis of corpus callosum excluded=False\n", + "\n", + "=== Blue_Sally_pretty.json ===\n", + " HP:0000122 Unilateral renal agenesis excluded=False\n", + " HP:0000122 Unilateral renal agenesis excluded=False\n", + " HP:0000813 Bicornuate uterus excluded=False\n", + "\n", + "=== Charm_Sally_pretty.json ===\n", + " HP:0010866 Abdominal wall defect excluded=False\n", + " HP:0001539 Omphalocele excluded=False\n", + " HP:0001539 Omphalocele excluded=False\n", + " HP:0001539 Omphalocele excluded=False\n", + "\n", + "=== Diva_Sally_pretty.json ===\n", + " HP:0030716 Acrania excluded=False\n", + "\n", + "=== Eclair_Sally_pretty.json ===\n", + " HP:0001627 Abnormal heart morphology excluded=False\n", + "\n", + "=== Gwen_Sally_pretty.json ===\n", + " HP:0000126 Hydronephrosis excluded=True\n", + "\n" ] } ], "source": [ - "for phrasing in [\"Dandy-Walker malformation\", \"Dandy Walker malformation\", \"Dandy-Walker syndrome\"]:\n", - " hits = hpo_cr.parse(phrasing)\n", - " print(f\"{phrasing!r:32s} -> {[(h.hpo_id, h.hpo_label) for h in hits]}\")" + "for path in sorted(DATA_DIR.glob(\"*_pretty.json\")):\n", + " fixture = json.loads(path.read_text())\n", + " text = fixture.get(\"finalize\", {}).get(\"generalComment\", {}).get(\"plain_text\")\n", + " if not text:\n", + " continue\n", + " print(f\"=== {path.name} ===\")\n", + " hits = hpo_cr.parse(text)\n", + " if not hits:\n", + " print(\" (no hits)\")\n", + " for hit in hits:\n", + " print(f\" {hit.hpo_id} {hit.hpo_label} excluded={hit.excluded}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "6eb579a9", + "metadata": {}, + "source": [ + "**What this survey shows:** every real anomaly actually mentioned in these six fixtures gets picked up - Charm Sally's omphalocele, Blue Sally's unilateral renal agenesis (twice - it's genuinely mentioned twice in the text), Diva Sally's acrania, Eclair Sally's cardiac anomaly (captured as the generic `Abnormal heart morphology` - the text only says \"suspicious for a cardiac anomaly,\" so there isn't a more specific term to extract), and Gwen Sally's correctly-excluded hydronephrosis. Apple Sally's Dandy-Walker miss and the negation-scope inconsistency above are the two real gaps in this set, not a pattern across every fixture - fenominal's actual failure rate here is low, but both gaps are exactly the kind of thing a systematic benchmark needs to catch rather than find by accident - a good next step for anyone evaluating text-mining alternatives." ] }, { @@ -880,14 +694,14 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "id": "13c7c760", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.077517Z", - "iopub.status.busy": "2026-07-27T16:39:45.077469Z", - "iopub.status.idle": "2026-07-27T16:39:45.084325Z", - "shell.execute_reply": "2026-07-27T16:39:45.084039Z" + "iopub.execute_input": "2026-07-28T13:28:16.930799Z", + "iopub.status.busy": "2026-07-28T13:28:16.930762Z", + "iopub.status.idle": "2026-07-28T13:28:16.937455Z", + "shell.execute_reply": "2026-07-28T13:28:16.937084Z" } }, "outputs": [ @@ -895,7225 +709,1591 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n" + "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Found 6 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=55.6% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=226.20000000000002mm, percentile=55.6%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=226.20000000000002mm, percentile=55.6%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=51.2% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=66.8mm, percentile=51.2%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=66.8mm, percentile=51.2%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=42.5% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=42.5%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=42.5%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=46.8% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.099999999999994mm, percentile=46.8%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.099999999999994mm, percentile=46.8%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Nuchal Fold\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Nuchal Fold has percentile=0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Nuchal Fold: value=10.0mm, percentile=0%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Nuchal Fold, value=10.0mm, percentile=0.0%, ga=, method=None\n", + "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Nuchal Fold' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Cerebellum\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Cerebellum has percentile=0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Cerebellum: value=30.0mm, percentile=0%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Cerebellum, value=30.0mm, percentile=0.0%, ga=, method=None\n", + "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Cerebellum' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n", + "INFO:prenatalppkt.etl.extractors.observer:Extracted 4 total TermBins across 1 fetuses\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Found 6 measurements\n" + "Fetus 1: 4 TermBins\n", + " HP:0034207 Abnormal fetal gastrointestinal system morphology normal=True (AC: 226.2 mm (55.6%) at 26w6d [Fetus 1])\n", + " HP:0000240 Abnormality of skull size normal=True (BPD: 66.8 mm (51.2%) at 26w6d [Fetus 1])\n", + " HP:0000240 Abnormality of skull size normal=True (HC: 250.0 mm (42.5%) at 26w6d [Fetus 1])\n", + " HP:0002823 Abnormal femur morphology normal=True (Femur: 50.1 mm (46.8%) at 27w0d [Fetus 1])\n" ] - }, + } + ], + "source": [ + "from prenatalppkt.etl.extractors import observer as observer_extractor\n", + "\n", + "bins_by_fetus = observer_extractor.extract_all_fetuses(raw)\n", + "for fetus_number, term_bins in bins_by_fetus.items():\n", + " print(f\"Fetus {fetus_number}: {len(term_bins)} TermBins\")\n", + " for tb in term_bins:\n", + " print(f\" {tb.hpo_id} {tb.hpo_label} normal={tb.normal} ({tb.description})\")" + ] + }, + { + "cell_type": "markdown", + "id": "41849833", + "metadata": {}, + "source": [ + "Only 4 of Apple Sally's 6 measurements produced a `TermBin` - Nuchal Fold and Cerebellum are optional measurements with no HPO mapping yet (a known, documented gap, not a bug: `OPTIONAL_MEASUREMENTS` in `etl/term_bin_factory.py`).\n", + "\n", + "### The section parsers - `etl/sections/`\n", + "\n", + "Six parsers, all sharing the same `parse_X(data, source_format)` shape. Tested in `test_clinical_impression.py`, `test_clinical_indication.py`, `test_pregnancy_dating.py`, `test_estimated_fetal_weight.py`, `test_fetal_ratios.py`, `test_fetal_anatomy.py`." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "ec18e8ec", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.944034Z", + "iopub.status.busy": "2026-07-28T13:28:16.943973Z", + "iopub.status.idle": "2026-07-28T13:28:16.949697Z", + "shell.execute_reply": "2026-07-28T13:28:16.949406Z" + } + }, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n" + "Dating: {'lmp': '0001-01-01', 'edd': None, 'assigned_edd': None, 'dating_method': None, 'ga_by_lmp': None, 'ga_by_ultrasound': None, 'assigned_ga': None, 'source_format': 'observer_json'}\n", + "\n", + "Clinical indication text: ''\n", + "\n", + "EFW: 1014.8 g, 55.6 th percentile, Hadlock (AC, FL, HC)\n", + "\n", + "Ratios: [('HC/AC', 1.105, True), ('FL/AC', 22.149, True), ('FL/BPD', 75, True)]\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=55.6% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=226.20000000000002mm, percentile=55.6%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=226.20000000000002mm, percentile=55.6%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=51.2% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=66.8mm, percentile=51.2%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=66.8mm, percentile=51.2%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=42.5% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=42.5%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=42.5%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=46.8% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.099999999999994mm, percentile=46.8%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.099999999999994mm, percentile=46.8%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Nuchal Fold\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Nuchal Fold has percentile=0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Nuchal Fold: value=10.0mm, percentile=0%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Nuchal Fold, value=10.0mm, percentile=0.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Nuchal Fold' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Cerebellum\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Cerebellum has percentile=0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Cerebellum: value=30.0mm, percentile=0%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Cerebellum, value=30.0mm, percentile=0.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Cerebellum' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'Femur', 'BPD', 'AC'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.etl.extractors.observer:Extracted 4 total TermBins across 1 fetuses\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetus 1: 4 TermBins\n", - " HP:0034207 Abnormal fetal gastrointestinal system morphology normal=True (AC: 226.2 mm (55.6%) at 26w6d [Fetus 1])\n", - " HP:0000240 Abnormality of skull size normal=True (BPD: 66.8 mm (51.2%) at 26w6d [Fetus 1])\n", - " HP:0000240 Abnormality of skull size normal=True (HC: 250.0 mm (42.5%) at 26w6d [Fetus 1])\n", - " HP:0002823 Abnormal femur morphology normal=True (Femur: 50.1 mm (46.8%) at 27w0d [Fetus 1])\n" - ] - } - ], - "source": [ - "from prenatalppkt.etl.extractors import observer as observer_extractor\n", - "\n", - "bins_by_fetus = observer_extractor.extract_all_fetuses(raw)\n", - "for fetus_number, term_bins in bins_by_fetus.items():\n", - " print(f\"Fetus {fetus_number}: {len(term_bins)} TermBins\")\n", - " for tb in term_bins:\n", - " print(f\" {tb.hpo_id} {tb.hpo_label} normal={tb.normal} ({tb.description})\")" - ] - }, - { - "cell_type": "markdown", - "id": "41849833", - "metadata": {}, - "source": [ - "Only 4 of Apple Sally's 6 measurements produced a `TermBin` - Nuchal Fold and Cerebellum are optional measurements with no HPO mapping yet (a known, documented gap, not a bug: `OPTIONAL_MEASUREMENTS` in `etl/term_bin_factory.py`).\n", - "\n", - "### The section parsers - `etl/sections/`\n", - "\n", - "Six parsers, all sharing the same `parse_X(data, source_format)` shape. Tested in `test_clinical_impression.py`, `test_clinical_indication.py`, `test_pregnancy_dating.py`, `test_estimated_fetal_weight.py`, `test_fetal_ratios.py`, `test_fetal_anatomy.py`." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "ec18e8ec", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.090430Z", - "iopub.status.busy": "2026-07-27T16:39:45.090380Z", - "iopub.status.idle": "2026-07-27T16:39:45.103874Z", - "shell.execute_reply": "2026-07-27T16:39:45.103512Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dating: {'lmp': '0001-01-01', 'edd': None, 'assigned_edd': None, 'dating_method': None, 'ga_by_lmp': None, 'ga_by_ultrasound': None, 'assigned_ga': None, 'source_format': 'observer_json'}\n", - "\n", - "Clinical indication text: ''\n", - "\n", - "EFW: 1014.8 g, 55.6 th percentile, Hadlock (AC, FL, HC)\n", - "\n", - "Ratios: [('HC/AC', 1.105, True), ('FL/AC', 22.149, True), ('FL/BPD', 75, True)]\n" - ] - } - ], - "source": [ - "from prenatalppkt.etl.sections import (\n", - " parse_pregnancy_dating,\n", - " parse_clinical_indication,\n", - " parse_estimated_fetal_weight,\n", - " parse_fetal_ratios,\n", - ")\n", - "\n", - "dating = parse_pregnancy_dating(raw, \"observer_json\")\n", - "print(\"Dating:\", {k: v for k, v in dating.items() if k != \"raw_data\"})\n", - "\n", - "indication = parse_clinical_indication(raw, \"observer_json\")\n", - "print(\"\\nClinical indication text:\", repr(indication[\"indication_text\"]))\n", - "\n", - "efw = parse_estimated_fetal_weight(raw, \"observer_json\")\n", - "print(\"\\nEFW:\", efw[\"efw_grams\"], \"g,\", efw[\"percentile\"], \"th percentile,\", efw[\"method\"])\n", - "\n", - "ratios = parse_fetal_ratios(raw, \"observer_json\")\n", - "print(\"\\nRatios:\", [(r[\"name\"], r[\"value\"], r[\"within_range\"]) for r in ratios[\"ratios\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "cf732ad6", - "metadata": {}, - "source": [ - "Two honest notes here, not bugs to fix right now: dating comes back mostly `None` (Apple Sally's fixture doesn't carry the fields this parser reads, and the builder falls back to reading gestational age out of a biometry `TermBin`'s own description instead - which is why Apple Sally's Phenopacket's GA is still correct despite this). Clinical indication text is empty even though the raw file *does* have an ICD-10 indication code (`Z36.1`) - the parser doesn't read that field today.\n", - "\n", - "### Fetal anatomy - already introduced in section 3, now the full picture" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "d6e5d899", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.104867Z", - "iopub.status.busy": "2026-07-27T16:39:45.104817Z", - "iopub.status.idle": "2026-07-27T16:39:45.107769Z", - "shell.execute_reply": "2026-07-27T16:39:45.107431Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Normal structures: 47\n", - "Abnormal structures: ['Head', 'Cerebellum']\n", - "Not visualized: ['Calvarium', 'BPD Level', 'Lateral Ventricles', 'Choroid Plexus', 'Cisterna Magna', 'Lungs', 'Distal Left Outflow', 'Distal Right Outflow', 'IVC', 'SVC', 'Spleen', 'Liver', 'Gall Bladder', 'Bowel', 'Lt Fingers', 'Rt Fingers', 'Lt Toes', 'Rt Toes']\n", - "Structured anomaly: [{'structure': 'Head', 'description': 'Dandy Walker', 'variant_type': 'Abnormal'}]\n", - "HPO terms found: [('HP:0045005', 'Neural tube defect', True)]\n" - ] - } - ], - "source": [ - "from prenatalppkt.etl.sections import parse_fetal_anatomy\n", - "\n", - "anatomy = parse_fetal_anatomy(raw, \"observer_json\", hpo_cr=hpo_cr)\n", - "print(\"Normal structures:\", len(anatomy[\"normal_structures\"]))\n", - "print(\"Abnormal structures:\", anatomy[\"abnormal_structures\"])\n", - "print(\"Not visualized:\", anatomy[\"not_visualized\"])\n", - "print(\"Structured anomaly:\", anatomy[\"anomalies\"])\n", - "print(\"HPO terms found:\", [(t.hpo_id, t.hpo_label, t.excluded) for t in anatomy[\"hpo_terms\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "de627831", - "metadata": {}, - "source": [ - "44 structures correctly read as normal, 2 correctly flagged abnormal (Head, Cerebellum), a real structured anomaly entry naming \"Dandy Walker\" directly - and still no `HP:0001305` in the HPO terms. Even this terse, 2-word structured form of the same finding doesn't get through fenominal. The free-text neural-tube-defect exclusion *is* caught correctly (`HP:0045005`, excluded=True) - so this isn't \"fenominal doesn't work,\" it's specifically this one term.\n", - "\n", - "### The builder - `builders/observer_phenopacket.py`\n", - "\n", - "Calls everything above, once per fetus, and assembles the final Phenopacket. Tested end-to-end in `test_observer_phenopacket.py` against all 5 real fixtures." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "aeec9301", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.108741Z", - "iopub.status.busy": "2026-07-27T16:39:45.108695Z", - "iopub.status.idle": "2026-07-27T16:39:45.189174Z", - "shell.execute_reply": "2026-07-27T16:39:45.188818Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Found 6 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=55.6% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=226.20000000000002mm, percentile=55.6%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=226.20000000000002mm, percentile=55.6%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=51.2% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=66.8mm, percentile=51.2%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=66.8mm, percentile=51.2%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=42.5% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=42.5%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=42.5%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=46.8% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.099999999999994mm, percentile=46.8%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.099999999999994mm, percentile=46.8%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Nuchal Fold\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Nuchal Fold has percentile=0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Nuchal Fold: value=10.0mm, percentile=0%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Nuchal Fold, value=10.0mm, percentile=0.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Nuchal Fold' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Cerebellum\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Cerebellum has percentile=0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Cerebellum: value=30.0mm, percentile=0%, ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Cerebellum, value=30.0mm, percentile=0.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Cerebellum' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'Femur', 'BPD', 'AC'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.etl.extractors.observer:Extracted 4 total TermBins across 1 fetuses\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Phenopacket id: applesally-fetus-1\n", - "Subject id: applesally-fetus-1\n", - "7 phenotypic features:\n", - " HP:0034207 Abnormal fetal gastrointestinal system morphology excluded=True (Biometry: AC: 226.2 mm (55.6%) at 26w6d [Fetus 1])\n", - " HP:0000240 Abnormality of skull size excluded=True (Biometry: BPD: 66.8 mm (51.2%) at 26w6d [Fetus 1])\n", - " HP:0002823 Abnormal femur morphology excluded=True (Biometry: Femur: 50.1 mm (46.8%) at 27w0d [Fetus 1])\n", - " HP:0000256 Macrocephaly excluded=True (Clinical impression: Macrocephaly)\n", - " HP:0002119 Ventriculomegaly excluded=True (Clinical impression: Ventriculomegaly)\n", - " HP:0001274 Agenesis of corpus callosum excluded=False (Clinical impression: Agenesis of corpus callosum)\n", - " HP:0045005 Neural tube defect excluded=True (Fetal anatomy)\n", - "\n", - "measurements: []\n" - ] - } - ], - "source": [ - "from datetime import datetime, timezone\n", - "from google.protobuf.timestamp_pb2 import Timestamp\n", - "from prenatalppkt.builders import build_observer_phenopacket\n", - "\n", - "now_ts = Timestamp()\n", - "now_ts.FromDatetime(datetime.now(tz=timezone.utc))\n", - "\n", - "pps = build_observer_phenopacket(raw, hpo_parser, now_ts, accession_id=\"applesally\")\n", - "apple_pp = pps[0]\n", - "print(\"Phenopacket id:\", apple_pp.id)\n", - "print(\"Subject id:\", apple_pp.subject.id)\n", - "print(f\"{len(apple_pp.phenotypic_features)} phenotypic features:\")\n", - "for pf in apple_pp.phenotypic_features:\n", - " print(f\" {pf.type.id} {pf.type.label} excluded={pf.excluded} ({pf.description})\")\n", - "print(\"\\nmeasurements:\", list(apple_pp.measurements))" - ] - }, - { - "cell_type": "markdown", - "id": "0a6ea9a2", - "metadata": {}, - "source": [ - "Seven features made it in: Apple Sally's 3 mappable biometry measurements (all normal, all excluded), 3 clinical-impression findings, 1 anatomy finding. Apple Sally's Dandy-Walker malformation is not among them - confirmed absent from the real, final Phenopacket, not just from an intermediate step.\n", - "\n", - "One more real, honest gap, visible right here: `measurements` is an empty list. `build_observer_phenopacket()` doesn't emit LOINC-coded raw `Measurement` objects at all today - only HPO features. (`prenatalppkt.ipynb`'s demo cell works around this by hand-adding `Measurement`s on top, using the raw `TermBin`s from the extractor cell above - that's a notebook-level patch, not something the builder does itself.)\n", - "\n", - "Also worth naming honestly: fenominal's negation detection isn't perfectly consistent either - Apple Sally's text says \"no evidence of ... agenesis of the corpus callosum,\" and while Macrocephaly and Ventriculomegaly both correctly come back `excluded=True`, Agenesis of corpus callosum comes back `excluded=False` from the exact same sentence." - ] - }, - { - "cell_type": "markdown", - "id": "33244342", - "metadata": {}, - "source": [ - "### A real bug this session found and fixed: twin exams\n", - "\n", - "Two fetuses in one exam each have their own anatomy data in the real Observer schema. The builder used to read only the first fetus's anatomy and copy it onto every fetus's Phenopacket - so a twin exam's second fetus silently inherited the first fetus's findings instead of its own. Confirmed with a synthetic two-fetus exam, one with a renal finding, one with a skull finding." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "3d517e5f", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.193069Z", - "iopub.status.busy": "2026-07-27T16:39:45.192964Z", - "iopub.status.idle": "2026-07-27T16:39:45.203469Z", - "shell.execute_reply": "2026-07-27T16:39:45.203077Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Found 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=65.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=65.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=220.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=220.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 2, scan_type=t2_t3_biometry\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Found 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=45.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=240.0mm, percentile=45.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=240.0mm, percentile=45.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=45.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=63.0mm, percentile=45.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=63.0mm, percentile=45.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=45.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=210.0mm, percentile=45.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=210.0mm, percentile=45.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=45.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=48.0mm, percentile=45.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=48.0mm, percentile=45.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.etl.extractors.observer:Extracted 8 total TermBins across 2 fetuses\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "twin-fetus-1: {'HP:0000104', 'HP:0034207', 'HP:0002823', 'HP:0000240'}\n", - "twin-fetus-2: {'HP:0034207', 'HP:0002823', 'HP:0030716', 'HP:0000240'}\n" - ] - } - ], - "source": [ - "fetus_1 = {\n", - " \"fetus\": {\"fetus_number\": 1},\n", - " \"measurements\": [\n", - " {\"label\": \"HC\", \"value\": 250.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " {\"label\": \"BPD\", \"value\": 65.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " {\"label\": \"AC\", \"value\": 220.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " {\"label\": \"Femur\", \"value\": 50.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " ],\n", - " \"anatomy\": [\n", - " {\"main\": {\"label\": \"Kidney\", \"anat_state\": \"Abnormal\"}, \"detail\": [],\n", - " \"anomalies\": [{\"description\": \"Renal agenesis\"}]}\n", - " ],\n", - "}\n", - "fetus_2 = {\n", - " \"fetus\": {\"fetus_number\": 2},\n", - " \"measurements\": [\n", - " {\"label\": \"HC\", \"value\": 240.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", - " {\"label\": \"BPD\", \"value\": 63.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", - " {\"label\": \"AC\", \"value\": 210.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", - " {\"label\": \"Femur\", \"value\": 48.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", - " ],\n", - " \"anatomy\": [\n", - " {\"main\": {\"label\": \"Skull\", \"anat_state\": \"Abnormal\"}, \"detail\": [],\n", - " \"anomalies\": [{\"description\": \"Acrania\"}]}\n", - " ],\n", - "}\n", - "twin_exam = {\"exam\": {\"fetus_count\": 2}, \"fetuses\": [fetus_1, fetus_2]}\n", - "\n", - "twin_pps = build_observer_phenopacket(twin_exam, hpo_parser, now_ts, accession_id=\"TWIN\")\n", - "for pp in twin_pps:\n", - " hpo_ids = {pf.type.id for pf in pp.phenotypic_features}\n", - " print(f\"{pp.id}: {hpo_ids}\")" - ] - }, - { - "cell_type": "markdown", - "id": "07c5d079", - "metadata": {}, - "source": [ - "Both sets share the normal-biometry findings (expected - their measurements are similar), but the anomaly is never shared: `HP:0000104` (Renal agenesis) appears only in fetus 1's set, `HP:0030716` (Acrania) appears only in fetus 2's - neither fetus inherits the other's anatomy finding. Confirmed fixed with a regression test (`test_twin_each_fetus_keeps_its_own_anatomy_finding`) that was checked against the pre-fix code and failed the expected way before the fix landed. ViewPoint HL7 had the identical bug, fixed the same way (section 5)." - ] - }, - { - "cell_type": "markdown", - "id": "ca8feb90", - "metadata": {}, - "source": [ - "### A real bug this session found and fixed: partial/targeted scans\n", - "\n", - "Some real exams only re-measure one or two things to check on a prior finding, rather than a full anatomy survey. The extractor used to require all four core measurements before it would parse *anything* for a fetus - so these targeted scans produced nothing at all, even when the measurements present were perfectly good." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "22c9d595", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.212758Z", - "iopub.status.busy": "2026-07-27T16:39:45.212665Z", - "iopub.status.idle": "2026-07-27T16:39:45.217072Z", - "shell.execute_reply": "2026-07-27T16:39:45.216779Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=unknown\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Found 2 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=240.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=240.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=50.0% (valid)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=63.0mm, percentile=50.0%, ga=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=63.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 2 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 2 TermBins for required measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'Femur', 'AC', 'BPD'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR:prenatalppkt.etl.term_bin_factory:Missing required biometry measurements: AC, Femur. Required measurements are: AC, BPD, Femur, HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:prenatalppkt.etl.extractors.observer:Fetus 1: Missing required biometry measurements: AC, Femur. Required measurements are: AC, BPD, Femur, HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.etl.extractors.observer:Extracted 2 total TermBins across 1 fetuses\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetus 1: 2 TermBins (used to be 0)\n", - " HP:0000240 Abnormality of skull size\n", - " HP:0000240 Abnormality of skull size\n" - ] - } - ], - "source": [ - "partial_scan = {\n", - " \"fetuses\": [{\n", - " \"fetus\": {\"fetus_number\": 1},\n", - " \"measurements\": [\n", - " {\"label\": \"HC\", \"value\": 240.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " {\"label\": \"BPD\", \"value\": 63.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", - " # No AC or Femur - a real, deliberate follow-up scan shape\n", - " ],\n", - " }]\n", - "}\n", - "partial_bins = observer_extractor.extract_all_fetuses(partial_scan)\n", - "for fetus_number, term_bins in partial_bins.items():\n", - " print(f\"Fetus {fetus_number}: {len(term_bins)} TermBins (used to be 0)\")\n", - " for tb in term_bins:\n", - " print(f\" {tb.hpo_id} {tb.hpo_label}\")" - ] - }, - { - "cell_type": "markdown", - "id": "c7b11c31", - "metadata": {}, - "source": [ - "Now returns the 2 measurements that are actually present, with a warning logged naming which of the four core measurements are missing - instead of discarding everything.\n", - "\n", - "### First trimester - Diva\n", - "\n", - "Diva is the one real first-trimester fixture: CRL only, no full biometry set. A different, deliberately simpler classification path (`ScanType.FIRST_TRIMESTER`)." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "a6f55166", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.220603Z", - "iopub.status.busy": "2026-07-27T16:39:45.220524Z", - "iopub.status.idle": "2026-07-27T16:39:45.224329Z", - "shell.execute_reply": "2026-07-27T16:39:45.224001Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=first_trimester\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=CRL, value=4.48mm, percentile=0.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0001511 - Intrauterine growth retardation\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0001511 - normal=False\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.etl.extractors.observer:Extracted 1 total TermBins across 1 fetuses\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Diva's features:\n", - " HP:0001511 Intrauterine growth retardation excluded=False\n", - " HP:0030716 Acrania excluded=False\n", - "\n", - "Diva's measurements: []\n" - ] - } - ], - "source": [ - "diva_raw = json.loads((DATA_DIR / \"Diva_Sally_pretty.json\").read_text())\n", - "diva_pps = build_observer_phenopacket(diva_raw, hpo_parser, now_ts, accession_id=\"divasally\")\n", - "diva_pp = diva_pps[0]\n", - "print(\"Diva's features:\")\n", - "for pf in diva_pp.phenotypic_features:\n", - " print(f\" {pf.type.id} {pf.type.label} excluded={pf.excluded}\")\n", - "print(\"\\nDiva's measurements:\", list(diva_pp.measurements))" - ] - }, - { - "cell_type": "markdown", - "id": "62e21690", - "metadata": {}, - "source": [ - "**This is the T1 gap you asked about specifically:** Diva's Phenopacket has real phenotypic features (Apple Sally's CRL-derived growth finding, plus Apple Sally's real clinical-impression finding) - the T1 pipeline itself works correctly. But `measurements` is empty, the same gap Apple Sally has. It's just most visible on Diva because CRL is Apple Sally's *only* measurement, so there's nothing else to distract from the empty list. This is not a T1-specific bug - it's the same \"builder doesn't emit `Measurement`s yet\" gap, on every fixture, T1 or not." - ] - }, - { - "cell_type": "markdown", - "id": "555d402e", - "metadata": {}, - "source": [ - "## 5. ViewPoint path, end to end\n", - "\n", - "A completely different raw shape (pipe-delimited HL7 segments instead of nested JSON), same three-layer pipeline. Using a real ViewPoint HL7 fixture with both biometry and a real anatomy finding." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "dcc03086", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.225369Z", - "iopub.status.busy": "2026-07-27T16:39:45.225332Z", - "iopub.status.idle": "2026-07-27T16:39:45.265342Z", - "shell.execute_reply": "2026-07-27T16:39:45.265064Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 25 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 25 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A|||||||||20260115140000...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|205^205.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 205^205.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 205.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 205.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=205.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&f...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|45^45%|%&percent^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|NM|SkullFetus.BiParietalDiameter^BPD|Fetus1|53^53.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.BiParietalDiameter^BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 53^53.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 53.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=value, value={'value': 53.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=53.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|NM|SkullFetus.VP_BiParietalDiameter_GA|Fetus1|149^21w 2d|d&weeks_days^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 149^21w 2d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.VP_BiParietalDiameter_Percentile|Fetus1|40^40%|%&percent^fmt...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 40^40%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=percentile, value=40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|AbdomenFetus.AbdominalCircumference^AC|Fetus1|168^168.0|mm&millimeters^...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.AbdominalCircumference^AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 168^168.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 168.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=value, value={'value': 168.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=168.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|NM|AbdomenFetus.VP_AbdominalCircumference_GA|Fetus1|151^21w 4d|d&weeks_day...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 151^21w 4d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|NM|AbdomenFetus.VP_AbdominalCircumference_Percentile|Fetus1|38^38%|%&perc...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 38^38%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=percentile, value=38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[10] Processing: OBX|11|NM|ExtremitiesFetus.FemurLength^FL|Fetus1|34^34.0|mm&millimeters^mm&milli...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.FemurLength^FL\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 34^34.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 34.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=value, value={'value': 34.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=34.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[11] Processing: OBX|12|NM|ExtremitiesFetus.VP_FemurLength_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[12] Processing: OBX|13|NM|ExtremitiesFetus.VP_FemurLength_Percentile|Fetus1|42^42%|%&percent^fmt...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 42^42%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=percentile, value=42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[13] Processing: OBX|14|ST|BrainFetus.BrainAppearance^Brain appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.BrainAppearance^Brain appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BrainAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BrainAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.BrainAppearance^Brain appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[14] Processing: OBX|15|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal|...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[15] Processing: OBX|16|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypo...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[16] Processing: OBX|17|ST|FaceFetus.FaceAppearance^Face appearance|Fetus1|normal|||||||||2026011...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: FaceFetus.FaceAppearance^Face appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FaceAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: FaceAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: FaceFetus.FaceAppearance^Face appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[17] Processing: OBX|18|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[18] Processing: OBX|19|ST|GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract ap...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=GastrointestinalTractAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: GastrointestinalTractAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[19] Processing: OBX|20|ST|SpineFetus.SpineAppearance^Spine appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SpineFetus.SpineAppearance^Spine appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=SpineAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: SpineAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: SpineFetus.SpineAppearance^Spine appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[20] Processing: OBX|21|ST|UrinaryTractFetus.BladderAppearance^Bladder appearance|Fetus1|normal||...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.BladderAppearance^Bladder appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BladderAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BladderAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.BladderAppearance^Bladder appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[21] Processing: OBX|22|ST|UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearanc...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=UrogenitalTractAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: UrogenitalTractAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[22] Processing: OBX|23|ST|UrinaryTractFetus.KidneyLAppearance^Left kidney appearance|Fetus1|norm...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyLAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyLAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[23] Processing: OBX|24|ST|UrinaryTractFetus.KidneyRAppearance^Right kidney appearance|Fetus1|nor...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyRAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyRAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[24] Processing: OBX|25|ST|RequestedProcedure.Indication^Indication^Order indication|1|Full secon...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: RequestedProcedure.Indication^Indication^Order indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: Full second trimester survey; anomaly follow-up\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Indication, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: RequestedProcedure.Indication^Indication^Order indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 205.0 mm -> 205.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=205.0mm, percentile=45.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 53.0 mm -> 53.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=53.0mm, percentile=40.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 168.0 mm -> 168.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=168.0mm, percentile=38.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 34.0 mm -> 34.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=34.0mm, percentile=42.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 4 TermBins total\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetus 1: 4 TermBins\n", - " HP:0000240 Abnormality of skull size | HC: 205.0 mm (45.0%) at 21w3d\n", - " HP:0000240 Abnormality of skull size | BPD: 53.0 mm (40.0%) at 21w2d\n", - " HP:0034207 Abnormal fetal gastrointestinal system morphology | AC: 168.0 mm (38.0%) at 21w4d\n", - " HP:0002823 Abnormal femur morphology | Femur: 34.0 mm (42.0%) at 21w3d\n" - ] - } - ], - "source": [ - "vp_data = (DATA_DIR / \"viewpoint_hl7_full_exam_test.txt\").read_text()\n", - "\n", - "from prenatalppkt.etl.extractors import viewpoint_hl7\n", - "\n", - "vp_bins = viewpoint_hl7.extract_all_fetuses(vp_data)\n", - "for num, tbs in vp_bins.items():\n", - " print(f\"Fetus {num}: {len(tbs)} TermBins\")\n", - " for tb in tbs:\n", - " print(\" \", tb.hpo_id, tb.hpo_label, \"|\", tb.description)" - ] - }, - { - "cell_type": "markdown", - "id": "ec75cb9a", - "metadata": {}, - "source": [ - "Same `TermBin` objects, same HPO+LOINC mapping - the only thing that changed is how the raw OBX segments got read. Now the anatomy side:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "cc4adbd3", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.297903Z", - "iopub.status.busy": "2026-07-27T16:39:45.297813Z", - "iopub.status.idle": "2026-07-27T16:39:45.299517Z", - "shell.execute_reply": "2026-07-27T16:39:45.299273Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Abnormal structures: ['Cerebellum']\n", - "Anomaly: [{'structure': 'Cerebellum', 'description': 'cerebellar hypoplasia', 'variant_type': 'Abnormal'}]\n", - "HPO terms: [('HP:0001321', 'Cerebellar hypoplasia')]\n" - ] - } - ], - "source": [ - "vp_anatomy = parse_fetal_anatomy(vp_data, \"viewpoint_hl7\", hpo_cr=hpo_cr)\n", - "print(\"Abnormal structures:\", vp_anatomy[\"abnormal_structures\"])\n", - "print(\"Anomaly:\", vp_anatomy[\"anomalies\"])\n", - "print(\"HPO terms:\", [(t.hpo_id, t.hpo_label) for t in vp_anatomy[\"hpo_terms\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "b52f739f", - "metadata": {}, - "source": [ - "\"Cerebellar hypoplasia\" - unlike Apple Sally's Dandy-Walker, this one *is* recognized correctly. Now the full build:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "d3023e9b", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.300574Z", - "iopub.status.busy": "2026-07-27T16:39:45.300532Z", - "iopub.status.idle": "2026-07-27T16:39:45.341497Z", - "shell.execute_reply": "2026-07-27T16:39:45.341057Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 25 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 25 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A|||||||||20260115140000...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|205^205.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 205^205.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 205.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 205.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=205.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&f...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|45^45%|%&percent^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|NM|SkullFetus.BiParietalDiameter^BPD|Fetus1|53^53.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.BiParietalDiameter^BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 53^53.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 53.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=value, value={'value': 53.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=53.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|NM|SkullFetus.VP_BiParietalDiameter_GA|Fetus1|149^21w 2d|d&weeks_days^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 149^21w 2d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.VP_BiParietalDiameter_Percentile|Fetus1|40^40%|%&percent^fmt...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 40^40%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=percentile, value=40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=40.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|AbdomenFetus.AbdominalCircumference^AC|Fetus1|168^168.0|mm&millimeters^...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.AbdominalCircumference^AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 168^168.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 168.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=value, value={'value': 168.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=168.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|NM|AbdomenFetus.VP_AbdominalCircumference_GA|Fetus1|151^21w 4d|d&weeks_day...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 151^21w 4d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|NM|AbdomenFetus.VP_AbdominalCircumference_Percentile|Fetus1|38^38%|%&perc...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 38^38%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=percentile, value=38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=38.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[10] Processing: OBX|11|NM|ExtremitiesFetus.FemurLength^FL|Fetus1|34^34.0|mm&millimeters^mm&milli...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.FemurLength^FL\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 34^34.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 34.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=value, value={'value': 34.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=34.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[11] Processing: OBX|12|NM|ExtremitiesFetus.VP_FemurLength_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_GA\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=ga\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (ga)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=ga, value=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[12] Processing: OBX|13|NM|ExtremitiesFetus.VP_FemurLength_Percentile|Fetus1|42^42%|%&percent^fmt...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 42^42%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=percentile, value=42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=42.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[13] Processing: OBX|14|ST|BrainFetus.BrainAppearance^Brain appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.BrainAppearance^Brain appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BrainAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BrainAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.BrainAppearance^Brain appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[14] Processing: OBX|15|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal|...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[15] Processing: OBX|16|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypo...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[16] Processing: OBX|17|ST|FaceFetus.FaceAppearance^Face appearance|Fetus1|normal|||||||||2026011...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: FaceFetus.FaceAppearance^Face appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FaceAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: FaceAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: FaceFetus.FaceAppearance^Face appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[17] Processing: OBX|18|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[18] Processing: OBX|19|ST|GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract ap...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=GastrointestinalTractAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: GastrointestinalTractAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[19] Processing: OBX|20|ST|SpineFetus.SpineAppearance^Spine appearance|Fetus1|normal|||||||||2026...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SpineFetus.SpineAppearance^Spine appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=SpineAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: SpineAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: SpineFetus.SpineAppearance^Spine appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[20] Processing: OBX|21|ST|UrinaryTractFetus.BladderAppearance^Bladder appearance|Fetus1|normal||...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.BladderAppearance^Bladder appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BladderAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BladderAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.BladderAppearance^Bladder appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[21] Processing: OBX|22|ST|UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearanc...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=UrogenitalTractAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: UrogenitalTractAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[22] Processing: OBX|23|ST|UrinaryTractFetus.KidneyLAppearance^Left kidney appearance|Fetus1|norm...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyLAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyLAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[23] Processing: OBX|24|ST|UrinaryTractFetus.KidneyRAppearance^Right kidney appearance|Fetus1|nor...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyRAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyRAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[24] Processing: OBX|25|ST|RequestedProcedure.Indication^Indication^Order indication|1|Full secon...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: RequestedProcedure.Indication^Indication^Order indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: Full second trimester survey; anomaly follow-up\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Indication, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: RequestedProcedure.Indication^Indication^Order indication\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 4 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 205.0 mm -> 205.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=205.0mm, percentile=45.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 53.0 mm -> 53.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=53.0mm, percentile=40.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for BPD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 168.0 mm -> 168.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=168.0mm, percentile=38.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for AC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 34.0 mm -> 34.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=34.0mm, percentile=42.0%, ga=, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for Femur\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 4 TermBins total\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id: full00099-fetus-1\n", - " HP:0000240 Abnormality of skull size excluded=True | Biometry: HC: 205.0 mm (45.0%) at 21w3d\n", - " HP:0034207 Abnormal fetal gastrointestinal system morphology excluded=True | Biometry: AC: 168.0 mm (38.0%) at 21w4d\n", - " HP:0002823 Abnormal femur morphology excluded=True | Biometry: Femur: 34.0 mm (42.0%) at 21w3d\n", - " HP:0001321 Cerebellar hypoplasia excluded=False | Fetal anatomy\n" - ] - } - ], - "source": [ - "from prenatalppkt.builders import build_viewpoint_phenopacket\n", - "\n", - "vp_pps = build_viewpoint_phenopacket(vp_data, hpo_parser, now_ts, accession_id=\"FULL00099\")\n", - "vp_pp = vp_pps[0]\n", - "print(\"id:\", vp_pp.id)\n", - "for pf in vp_pp.phenotypic_features:\n", - " print(\" \", pf.type.id, pf.type.label, \"excluded=\"+str(pf.excluded), \"|\", pf.description)" - ] - }, - { - "cell_type": "markdown", - "id": "550e2c41", - "metadata": {}, - "source": [ - "Only 4 features, not 5 - BPD is missing even though the extractor found it. HC and BPD both map to the same HPO term (`HP:0000240`, \"Abnormality of skull size\"), and the builder's HPO-id dedup keeps only the first occurrence. Expected, not a bug - the same dedup Apple Sally's Observer builder uses too.\n", - "\n", - "### The twin-anatomy bug, ViewPoint's independent copy\n", - "\n", - "ViewPoint HL7 had the exact same bug as Observer, found separately: `_parse_viewpoint_hl7_anatomy`'s own code comment claimed anatomy OBX segments \"aren't per-fetus tagged the same way biometry's are\" - checked directly against a real fixture, and that claim was wrong. Anatomy segments carry the same `Fetus1`/`Fetus2` sub-id biometry segments do." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "2959c6ca", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.373431Z", - "iopub.status.busy": "2026-07-27T16:39:45.373351Z", - "iopub.status.idle": "2026-07-27T16:39:45.403938Z", - "shell.execute_reply": "2026-07-27T16:39:45.403623Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 10 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 10 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 175^175.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 175.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 175.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=175.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 50^50%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypop...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|ST|Fetus.Identifier^Fetus Identifier|Fetus2|B...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: B\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.HeadCircumference^HC|Fetus2|170^170.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 170^170.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 170.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus2, measurement=HC, field=value, value={'value': 170.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=170.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus2|45^45%|%&percent^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus2, measurement=HC, field=percentile, value=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|ST|BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearanc...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=LateralVentricleLAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: LateralVentricleLAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|ST|BrainFetus.LateralVentricleLDetails^Left lateral ventricle details|Fet...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.LateralVentricleLDetails^Left lateral ventricle details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: ventriculomegaly\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=LateralVentricleLDetails, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: LateralVentricleLDetails\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.LateralVentricleLDetails^Left lateral ventricle details\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 2 fetus records\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 1 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus2: 1 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 170.0, 'unit': 'mm', 'percentile': 45.0, 'ga': None, 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 175.0 mm -> 175.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=175.0mm, percentile=50.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 1 TermBins total\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 170.0, 'unit': 'mm', 'percentile': 45.0, 'ga': None, 'method': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 170.0 mm -> 170.0 mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=170.0mm, percentile=45.0%, ga=None, method=None\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 1 TermBins total\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "twin-fetus-1: {'HP:0001321', 'HP:0000240'}\n", - "twin-fetus-2: {'HP:0002119', 'HP:0000240'}\n" - ] - } - ], - "source": [ - "twin_hl7 = r\"\"\"\n", - "MSH|^~\\&|ViewPoint|Hospital|||20260115120000||ORU^R01|123456|P|2.4\n", - "OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A\n", - "OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&millimeters\n", - "OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&formatted\n", - "OBX|4|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal\n", - "OBX|5|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypoplasia\n", - "OBX|6|ST|Fetus.Identifier^Fetus Identifier|Fetus2|B\n", - "OBX|7|NM|SkullFetus.HeadCircumference^HC|Fetus2|170^170.0|mm&millimeters^mm&millimeters\n", - "OBX|8|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus2|45^45%|%&percent^fmt&formatted\n", - "OBX|9|ST|BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance|Fetus2|abnormal\n", - "OBX|10|ST|BrainFetus.LateralVentricleLDetails^Left lateral ventricle details|Fetus2|ventriculomegaly\n", - "\"\"\"\n", - "twin_vp_pps = build_viewpoint_phenopacket(twin_hl7, hpo_parser, now_ts, accession_id=\"TWIN\")\n", - "for pp in twin_vp_pps:\n", - " hpo_ids = {pf.type.id for pf in pp.phenotypic_features}\n", - " print(f\"{pp.id}: {hpo_ids}\")" - ] - }, - { - "cell_type": "markdown", - "id": "63a11f95", - "metadata": {}, - "source": [ - "Fetus 1's cerebellar hypoplasia (`HP:0001321`) and fetus 2's ventriculomegaly (`HP:0002119`) each stay with their own fetus. Confirmed fixed, same pattern as the Observer fix.\n", - "\n", - "### A fenominal gap that needed its own synthetic message to prove\n", - "\n", - "The real ViewPoint anatomy fixture that has \"mild aortic arch narrowing\" has no biometry data at all, so it produces zero Phenopackets through the full builder - there was no way to prove the gap survives to a real final Phenopacket without building one by hand." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "84789133", - "metadata": { - "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.414109Z", - "iopub.status.busy": "2026-07-27T16:39:45.414025Z", - "iopub.status.idle": "2026-07-27T16:39:45.436820Z", - "shell.execute_reply": "2026-07-27T16:39:45.436436Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 5 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 5 OBX segments\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&mill...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 175^175.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 175.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 175.0, 'unit': 'mm'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=175.0 unit=mm\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 50^50%\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=50.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|abnormal...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|ST|ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details|F...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details\n" - ] - }, + } + ], + "source": [ + "from prenatalppkt.etl.sections import (\n", + " parse_pregnancy_dating,\n", + " parse_clinical_indication,\n", + " parse_estimated_fetal_weight,\n", + " parse_fetal_ratios,\n", + ")\n", + "\n", + "dating = parse_pregnancy_dating(raw, \"observer_json\")\n", + "print(\"Dating:\", {k: v for k, v in dating.items() if k != \"raw_data\"})\n", + "\n", + "indication = parse_clinical_indication(raw, \"observer_json\")\n", + "print(\"\\nClinical indication text:\", repr(indication[\"indication_text\"]))\n", + "\n", + "efw = parse_estimated_fetal_weight(raw, \"observer_json\")\n", + "print(\"\\nEFW:\", efw[\"efw_grams\"], \"g,\", efw[\"percentile\"], \"th percentile,\", efw[\"method\"])\n", + "\n", + "ratios = parse_fetal_ratios(raw, \"observer_json\")\n", + "print(\"\\nRatios:\", [(r[\"name\"], r[\"value\"], r[\"within_range\"]) for r in ratios[\"ratios\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "cf732ad6", + "metadata": {}, + "source": [ + "Two honest notes here, not bugs to fix right now: dating comes back mostly `None` (Apple Sally's fixture doesn't carry the fields this parser reads, and the builder falls back to reading gestational age out of a biometry `TermBin`'s own description instead - which is why Apple Sally's Phenopacket's GA is still correct despite this). Clinical indication text is empty even though the raw file *does* have an ICD-10 indication code (`Z36.1`) - the parser doesn't read that field today.\n", + "\n", + "### Fetal anatomy - already introduced in section 3, now the full picture" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "d6e5d899", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.950507Z", + "iopub.status.busy": "2026-07-28T13:28:16.950463Z", + "iopub.status.idle": "2026-07-28T13:28:16.952969Z", + "shell.execute_reply": "2026-07-28T13:28:16.952700Z" + } + }, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n" + "Normal structures: 47\n", + "Abnormal structures: ['Head', 'Cerebellum']\n", + "Not visualized: ['Calvarium', 'BPD Level', 'Lateral Ventricles', 'Choroid Plexus', 'Cisterna Magna', 'Lungs', 'Distal Left Outflow', 'Distal Right Outflow', 'IVC', 'SVC', 'Spleen', 'Liver', 'Gall Bladder', 'Bowel', 'Lt Fingers', 'Rt Fingers', 'Lt Toes', 'Rt Toes']\n", + "Structured anomaly: [{'structure': 'Head', 'description': 'Dandy Walker', 'variant_type': 'Abnormal'}]\n", + "HPO terms found: [('HP:0045005', 'Neural tube defect', True)]\n" ] - }, + } + ], + "source": [ + "from prenatalppkt.etl.sections import parse_fetal_anatomy\n", + "\n", + "anatomy = parse_fetal_anatomy(raw, \"observer_json\", hpo_cr=hpo_cr)\n", + "print(\"Normal structures:\", len(anatomy[\"normal_structures\"]))\n", + "print(\"Abnormal structures:\", anatomy[\"abnormal_structures\"])\n", + "print(\"Not visualized:\", anatomy[\"not_visualized\"])\n", + "print(\"Structured anomaly:\", anatomy[\"anomalies\"])\n", + "print(\"HPO terms found:\", [(t.hpo_id, t.hpo_label, t.excluded) for t in anatomy[\"hpo_terms\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "de627831", + "metadata": {}, + "source": [ + "44 structures correctly read as normal, 2 correctly flagged abnormal (Head, Cerebellum), a real structured anomaly entry naming \"Dandy Walker\" directly - and still no `HP:0001305` in the HPO terms. Even this terse, 2-word structured form of the same finding doesn't get through fenominal. The free-text neural-tube-defect exclusion *is* caught correctly (`HP:0045005`, excluded=True) - so this isn't \"fenominal doesn't work,\" it's specifically this one term.\n", + "\n", + "### The builder - `builders/observer_phenopacket.py`\n", + "\n", + "Calls everything above, once per fetus, and assembles the final Phenopacket. Tested end-to-end in `test_observer_phenopacket.py` against all 5 real fixtures." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "aeec9301", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.953854Z", + "iopub.status.busy": "2026-07-28T13:28:16.953819Z", + "iopub.status.idle": "2026-07-28T13:28:16.990978Z", + "shell.execute_reply": "2026-07-28T13:28:16.990678Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: mild aortic arch narrowing\n" + "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Found 6 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=55.6% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=226.20000000000002mm, percentile=55.6%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=226.20000000000002mm, percentile=55.6%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=51.2% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=66.8mm, percentile=51.2%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=66.8mm, percentile=51.2%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=42.5% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=42.5%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=42.5%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=46.8% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.099999999999994mm, percentile=46.8%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.099999999999994mm, percentile=46.8%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Nuchal Fold\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Nuchal Fold has percentile=0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Nuchal Fold: value=10.0mm, percentile=0%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Nuchal Fold, value=10.0mm, percentile=0.0%, ga=, method=None\n", + "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Nuchal Fold' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Cerebellum\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Cerebellum has percentile=0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Cerebellum: value=30.0mm, percentile=0%, ga=\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Cerebellum, value=30.0mm, percentile=0.0%, ga=, method=None\n", + "WARNING:prenatalppkt.etl.term_bin_factory:No HPO mapping for optional measurement 'Cerebellum' - skipping. TODO(@VarenyaJ): Add HPO terms when available\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n", + "INFO:prenatalppkt.etl.extractors.observer:Extracted 4 total TermBins across 1 fetuses\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ThoracicDescAortaDetails, type=value\n" + "Phenopacket id: applesally-fetus-1\n", + "Subject id: applesally-fetus-1\n", + "7 phenotypic features:\n", + " HP:0034207 Abnormal fetal gastrointestinal system morphology excluded=True (Biometry: AC: 226.2 mm (55.6%) at 26w6d [Fetus 1])\n", + " HP:0000240 Abnormality of skull size excluded=True (Biometry: BPD: 66.8 mm (51.2%) at 26w6d [Fetus 1])\n", + " HP:0002823 Abnormal femur morphology excluded=True (Biometry: Femur: 50.1 mm (46.8%) at 27w0d [Fetus 1])\n", + " HP:0000256 Macrocephaly excluded=True (Clinical impression: Macrocephaly)\n", + " HP:0002119 Ventriculomegaly excluded=True (Clinical impression: Ventriculomegaly)\n", + " HP:0001274 Agenesis of corpus callosum excluded=False (Clinical impression: Agenesis of corpus callosum)\n", + " HP:0045005 Neural tube defect excluded=True (Fetal anatomy)\n", + "\n", + "measurements: []\n" ] - }, + } + ], + "source": [ + "from datetime import datetime, timezone\n", + "from google.protobuf.timestamp_pb2 import Timestamp\n", + "from prenatalppkt.builders import build_observer_phenopacket\n", + "\n", + "now_ts = Timestamp()\n", + "now_ts.FromDatetime(datetime.now(tz=timezone.utc))\n", + "\n", + "pps = build_observer_phenopacket(raw, hpo_parser, now_ts, accession_id=\"applesally\")\n", + "apple_pp = pps[0]\n", + "print(\"Phenopacket id:\", apple_pp.id)\n", + "print(\"Subject id:\", apple_pp.subject.id)\n", + "print(f\"{len(apple_pp.phenotypic_features)} phenotypic features:\")\n", + "for pf in apple_pp.phenotypic_features:\n", + " print(f\" {pf.type.id} {pf.type.label} excluded={pf.excluded} ({pf.description})\")\n", + "print(\"\\nmeasurements:\", list(apple_pp.measurements))" + ] + }, + { + "cell_type": "markdown", + "id": "0a6ea9a2", + "metadata": {}, + "source": [ + "Seven features made it in: Apple Sally's 3 mappable biometry measurements (all normal, all excluded), 3 clinical-impression findings, 1 anatomy finding. Apple Sally's Dandy-Walker malformation is not among them - confirmed absent from the real, final Phenopacket, not just from an intermediate step.\n", + "\n", + "One more real, honest gap, visible right here: `measurements` is an empty list. `build_observer_phenopacket()` doesn't emit LOINC-coded raw `Measurement` objects at all today - only HPO features. (`prenatalppkt.ipynb`'s demo cell works around this by hand-adding `Measurement`s on top, using the raw `TermBin`s from the extractor cell above - that's a notebook-level patch, not something the builder does itself.)\n", + "\n", + "Also worth naming honestly: fenominal's negation detection isn't perfectly consistent either - Apple Sally's text says \"no evidence of ... agenesis of the corpus callosum,\" and while Macrocephaly and Ventriculomegaly both correctly come back `excluded=True`, Agenesis of corpus callosum comes back `excluded=False` from the exact same sentence." + ] + }, + { + "cell_type": "markdown", + "id": "33244342", + "metadata": {}, + "source": [ + "### A real bug this session found and fixed: twin exams\n", + "\n", + "Two fetuses in one exam each have their own anatomy data in the real Observer schema. The builder used to read only the first fetus's anatomy and copy it onto every fetus's Phenopacket - so a twin exam's second fetus silently inherited the first fetus's findings instead of its own. Confirmed with a synthetic two-fetus exam, one with a renal finding, one with a skull finding." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "3d517e5f", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:16.994695Z", + "iopub.status.busy": "2026-07-28T13:28:16.994605Z", + "iopub.status.idle": "2026-07-28T13:28:17.004751Z", + "shell.execute_reply": "2026-07-28T13:28:17.004437Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ThoracicDescAortaDetails\n" + "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=t2_t3_biometry\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Found 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=250.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=250.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=65.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=65.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=220.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=220.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=50.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=50.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'Femur', 'AC', 'BPD', 'HC'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 2, scan_type=t2_t3_biometry\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Found 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=45.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=240.0mm, percentile=45.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=240.0mm, percentile=45.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=45.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=63.0mm, percentile=45.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=63.0mm, percentile=45.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:AC has percentile=45.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for AC: value=210.0mm, percentile=45.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=210.0mm, percentile=45.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Femur has percentile=45.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for Femur: value=48.0mm, percentile=45.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=48.0mm, percentile=45.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 4 measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 4 TermBins for required measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: AC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: Femur\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'Femur', 'AC', 'BPD', 'HC'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'BPD', 'AC', 'Femur'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:All required measurements present\n", + "INFO:prenatalppkt.etl.extractors.observer:Extracted 8 total TermBins across 2 fetuses\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details\n" + "twin-fetus-1: {'HP:0034207', 'HP:0002823', 'HP:0000104', 'HP:0000240'}\n", + "twin-fetus-2: {'HP:0034207', 'HP:0002823', 'HP:0030716', 'HP:0000240'}\n" ] - }, + } + ], + "source": [ + "fetus_1 = {\n", + " \"fetus\": {\"fetus_number\": 1},\n", + " \"measurements\": [\n", + " {\"label\": \"HC\", \"value\": 250.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " {\"label\": \"BPD\", \"value\": 65.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " {\"label\": \"AC\", \"value\": 220.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " {\"label\": \"Femur\", \"value\": 50.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " ],\n", + " \"anatomy\": [\n", + " {\"main\": {\"label\": \"Kidney\", \"anat_state\": \"Abnormal\"}, \"detail\": [],\n", + " \"anomalies\": [{\"description\": \"Renal agenesis\"}]}\n", + " ],\n", + "}\n", + "fetus_2 = {\n", + " \"fetus\": {\"fetus_number\": 2},\n", + " \"measurements\": [\n", + " {\"label\": \"HC\", \"value\": 240.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", + " {\"label\": \"BPD\", \"value\": 63.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", + " {\"label\": \"AC\", \"value\": 210.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", + " {\"label\": \"Femur\", \"value\": 48.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 45.0},\n", + " ],\n", + " \"anatomy\": [\n", + " {\"main\": {\"label\": \"Skull\", \"anat_state\": \"Abnormal\"}, \"detail\": [],\n", + " \"anomalies\": [{\"description\": \"Acrania\"}]}\n", + " ],\n", + "}\n", + "twin_exam = {\"exam\": {\"fetus_count\": 2}, \"fetuses\": [fetus_1, fetus_2]}\n", + "\n", + "twin_pps = build_observer_phenopacket(twin_exam, hpo_parser, now_ts, accession_id=\"TWIN\")\n", + "for pp in twin_pps:\n", + " hpo_ids = {pf.type.id for pf in pp.phenotypic_features}\n", + " print(f\"{pp.id}: {hpo_ids}\")" + ] + }, + { + "cell_type": "markdown", + "id": "07c5d079", + "metadata": {}, + "source": [ + "Both sets share the normal-biometry findings (expected - their measurements are similar), but the anomaly is never shared: `HP:0000104` (Renal agenesis) appears only in fetus 1's set, `HP:0030716` (Acrania) appears only in fetus 2's - neither fetus inherits the other's anatomy finding. Confirmed fixed with a regression test (`test_twin_each_fetus_keeps_its_own_anatomy_finding`) that was checked against the pre-fix code and failed the expected way before the fix landed. ViewPoint HL7 had the identical bug, fixed the same way (section 5)." + ] + }, + { + "cell_type": "markdown", + "id": "ca8feb90", + "metadata": {}, + "source": [ + "### A real bug this session found and fixed: partial/targeted scans\n", + "\n", + "Some real exams only re-measure one or two things to check on a prior finding, rather than a full anatomy survey. The extractor used to require all four core measurements before it would parse *anything* for a fetus - so these targeted scans produced nothing at all, even when the measurements present were perfectly good." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "22c9d595", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.013797Z", + "iopub.status.busy": "2026-07-28T13:28:17.013711Z", + "iopub.status.idle": "2026-07-28T13:28:17.017967Z", + "shell.execute_reply": "2026-07-28T13:28:17.017659Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n" + "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=unknown\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Found 2 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.observer:HC has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for HC: value=240.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=240.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.observer:BPD has percentile=50.0% (valid)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Creating TermBin for BPD: value=63.0mm, percentile=50.0%, ga=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=63.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Successfully parsed 2 measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Validating 2 TermBins for required measurements\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: HC\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Found measurement: BPD\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Present: {'BPD', 'HC'}\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Required: {'HC', 'BPD', 'AC', 'Femur'}\n", + "ERROR:prenatalppkt.etl.term_bin_factory:Missing required biometry measurements: AC, Femur. Required measurements are: AC, BPD, Femur, HC\n", + "WARNING:prenatalppkt.etl.extractors.observer:Fetus 1: Missing required biometry measurements: AC, Femur. Required measurements are: AC, BPD, Femur, HC\n", + "INFO:prenatalppkt.etl.extractors.observer:Extracted 2 total TermBins across 1 fetuses\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n" + "Fetus 1: 2 TermBins (used to be 0)\n", + " HP:0000240 Abnormality of skull size\n", + " HP:0000240 Abnormality of skull size\n" ] - }, + } + ], + "source": [ + "partial_scan = {\n", + " \"fetuses\": [{\n", + " \"fetus\": {\"fetus_number\": 1},\n", + " \"measurements\": [\n", + " {\"label\": \"HC\", \"value\": 240.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " {\"label\": \"BPD\", \"value\": 63.0, \"unit_of_measure\": \"mm\", \"calculated_percentile\": 50.0},\n", + " # No AC or Femur - a real, deliberate follow-up scan shape\n", + " ],\n", + " }]\n", + "}\n", + "partial_bins = observer_extractor.extract_all_fetuses(partial_scan)\n", + "for fetus_number, term_bins in partial_bins.items():\n", + " print(f\"Fetus {fetus_number}: {len(term_bins)} TermBins (used to be 0)\")\n", + " for tb in term_bins:\n", + " print(f\" {tb.hpo_id} {tb.hpo_label}\")" + ] + }, + { + "cell_type": "markdown", + "id": "c7b11c31", + "metadata": {}, + "source": [ + "Now returns the 2 measurements that are actually present, with a warning logged naming which of the four core measurements are missing - instead of discarding everything.\n", + "\n", + "### First trimester - Diva\n", + "\n", + "Diva is the one real first-trimester fixture: CRL only, no full biometry set. A different, deliberately simpler classification path (`ScanType.FIRST_TRIMESTER`)." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "a6f55166", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.021292Z", + "iopub.status.busy": "2026-07-28T13:28:17.021227Z", + "iopub.status.idle": "2026-07-28T13:28:17.025066Z", + "shell.execute_reply": "2026-07-28T13:28:17.024761Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 1 measurements\n" + "DEBUG:prenatalppkt.etl.extractors.observer:Starting Observer JSON extraction (multi-fetus)\n", + "DEBUG:prenatalppkt.etl.extractors.observer:Processing fetus 1, scan_type=first_trimester\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=CRL, value=4.48mm, percentile=0.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0001511 - Intrauterine growth retardation\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0001511 - normal=False\n", + "INFO:prenatalppkt.etl.extractors.observer:Extracted 1 total TermBins across 1 fetuses\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n" + "Diva's features:\n", + " HP:0001511 Intrauterine growth retardation excluded=False\n", + " HP:0030716 Acrania excluded=False\n", + "\n", + "Diva's measurements: []\n" ] - }, + } + ], + "source": [ + "diva_raw = json.loads((DATA_DIR / \"Diva_Sally_pretty.json\").read_text())\n", + "diva_pps = build_observer_phenopacket(diva_raw, hpo_parser, now_ts, accession_id=\"divasally\")\n", + "diva_pp = diva_pps[0]\n", + "print(\"Diva's features:\")\n", + "for pf in diva_pp.phenotypic_features:\n", + " print(f\" {pf.type.id} {pf.type.label} excluded={pf.excluded}\")\n", + "print(\"\\nDiva's measurements:\", list(diva_pp.measurements))" + ] + }, + { + "cell_type": "markdown", + "id": "62e21690", + "metadata": {}, + "source": [ + "**This is the T1 gap you asked about specifically:** Diva's Phenopacket has real phenotypic features (Apple Sally's CRL-derived growth finding, plus Apple Sally's real clinical-impression finding) - the T1 pipeline itself works correctly. But `measurements` is empty, the same gap Apple Sally has. It's just most visible on Diva because CRL is Apple Sally's *only* measurement, so there's nothing else to distract from the empty list. This is not a T1-specific bug - it's the same \"builder doesn't emit `Measurement`s yet\" gap, on every fixture, T1 or not." + ] + }, + { + "cell_type": "markdown", + "id": "555d402e", + "metadata": {}, + "source": [ + "## 5. ViewPoint path, end to end\n", + "\n", + "A completely different raw shape (pipe-delimited HL7 segments instead of nested JSON), same three-layer pipeline. Using a real ViewPoint HL7 fixture with both biometry and a real anatomy finding." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "dcc03086", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.026068Z", + "iopub.status.busy": "2026-07-28T13:28:17.026027Z", + "iopub.status.idle": "2026-07-28T13:28:17.065833Z", + "shell.execute_reply": "2026-07-28T13:28:17.065518Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n" + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 25 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 25 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A|||||||||20260115140000...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|205^205.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 205^205.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 205.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 205.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=205.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&f...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|45^45%|%&percent^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|NM|SkullFetus.BiParietalDiameter^BPD|Fetus1|53^53.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.BiParietalDiameter^BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 53^53.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 53.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=value, value={'value': 53.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=53.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|NM|SkullFetus.VP_BiParietalDiameter_GA|Fetus1|149^21w 2d|d&weeks_days^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 149^21w 2d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.VP_BiParietalDiameter_Percentile|Fetus1|40^40%|%&percent^fmt...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 40^40%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=percentile, value=40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|AbdomenFetus.AbdominalCircumference^AC|Fetus1|168^168.0|mm&millimeters^...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.AbdominalCircumference^AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 168^168.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 168.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=value, value={'value': 168.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=168.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|NM|AbdomenFetus.VP_AbdominalCircumference_GA|Fetus1|151^21w 4d|d&weeks_day...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 151^21w 4d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|NM|AbdomenFetus.VP_AbdominalCircumference_Percentile|Fetus1|38^38%|%&perc...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 38^38%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=percentile, value=38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[10] Processing: OBX|11|NM|ExtremitiesFetus.FemurLength^FL|Fetus1|34^34.0|mm&millimeters^mm&milli...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.FemurLength^FL\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 34^34.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 34.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=value, value={'value': 34.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=34.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[11] Processing: OBX|12|NM|ExtremitiesFetus.VP_FemurLength_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[12] Processing: OBX|13|NM|ExtremitiesFetus.VP_FemurLength_Percentile|Fetus1|42^42%|%&percent^fmt...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 42^42%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=percentile, value=42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[13] Processing: OBX|14|ST|BrainFetus.BrainAppearance^Brain appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.BrainAppearance^Brain appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BrainAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BrainAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.BrainAppearance^Brain appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[14] Processing: OBX|15|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal|...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[15] Processing: OBX|16|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypo...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[16] Processing: OBX|17|ST|FaceFetus.FaceAppearance^Face appearance|Fetus1|normal|||||||||2026011...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: FaceFetus.FaceAppearance^Face appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FaceAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: FaceAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: FaceFetus.FaceAppearance^Face appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[17] Processing: OBX|18|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[18] Processing: OBX|19|ST|GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract ap...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=GastrointestinalTractAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: GastrointestinalTractAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[19] Processing: OBX|20|ST|SpineFetus.SpineAppearance^Spine appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SpineFetus.SpineAppearance^Spine appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=SpineAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: SpineAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: SpineFetus.SpineAppearance^Spine appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[20] Processing: OBX|21|ST|UrinaryTractFetus.BladderAppearance^Bladder appearance|Fetus1|normal||...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.BladderAppearance^Bladder appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BladderAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BladderAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.BladderAppearance^Bladder appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[21] Processing: OBX|22|ST|UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearanc...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=UrogenitalTractAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: UrogenitalTractAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[22] Processing: OBX|23|ST|UrinaryTractFetus.KidneyLAppearance^Left kidney appearance|Fetus1|norm...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyLAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyLAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[23] Processing: OBX|24|ST|UrinaryTractFetus.KidneyRAppearance^Right kidney appearance|Fetus1|nor...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyRAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyRAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[24] Processing: OBX|25|ST|RequestedProcedure.Indication^Indication^Order indication|1|Full secon...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: RequestedProcedure.Indication^Indication^Order indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: 1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: Full second trimester survey; anomaly follow-up\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Indication, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: RequestedProcedure.Indication^Indication^Order indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 205.0 mm -> 205.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=205.0mm, percentile=45.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 53.0 mm -> 53.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=53.0mm, percentile=40.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 168.0 mm -> 168.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=168.0mm, percentile=38.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 34.0 mm -> 34.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=34.0mm, percentile=42.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 4 TermBins total\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n" + "Fetus 1: 4 TermBins\n", + " HP:0000240 Abnormality of skull size | HC: 205.0 mm (45.0%) at 21w3d\n", + " HP:0000240 Abnormality of skull size | BPD: 53.0 mm (40.0%) at 21w2d\n", + " HP:0034207 Abnormal fetal gastrointestinal system morphology | AC: 168.0 mm (38.0%) at 21w4d\n", + " HP:0002823 Abnormal femur morphology | Femur: 34.0 mm (42.0%) at 21w3d\n" ] - }, + } + ], + "source": [ + "vp_data = (DATA_DIR / \"viewpoint_hl7_full_exam_test.txt\").read_text()\n", + "\n", + "from prenatalppkt.etl.extractors import viewpoint_hl7\n", + "\n", + "vp_bins = viewpoint_hl7.extract_all_fetuses(vp_data)\n", + "for num, tbs in vp_bins.items():\n", + " print(f\"Fetus {num}: {len(tbs)} TermBins\")\n", + " for tb in tbs:\n", + " print(\" \", tb.hpo_id, tb.hpo_label, \"|\", tb.description)" + ] + }, + { + "cell_type": "markdown", + "id": "ec75cb9a", + "metadata": {}, + "source": [ + "Same `TermBin` objects, same HPO+LOINC mapping - the only thing that changed is how the raw OBX segments got read. Now the anatomy side:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "cc4adbd3", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.099898Z", + "iopub.status.busy": "2026-07-28T13:28:17.099828Z", + "iopub.status.idle": "2026-07-28T13:28:17.101438Z", + "shell.execute_reply": "2026-07-28T13:28:17.101152Z" + } + }, + "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 175.0 mm -> 175.0 mm\n" + "Abnormal structures: ['Cerebellum']\n", + "Anomaly: [{'structure': 'Cerebellum', 'description': 'cerebellar hypoplasia', 'variant_type': 'Abnormal'}]\n", + "HPO terms: [('HP:0001321', 'Cerebellar hypoplasia')]\n" ] - }, + } + ], + "source": [ + "vp_anatomy = parse_fetal_anatomy(vp_data, \"viewpoint_hl7\", hpo_cr=hpo_cr)\n", + "print(\"Abnormal structures:\", vp_anatomy[\"abnormal_structures\"])\n", + "print(\"Anomaly:\", vp_anatomy[\"anomalies\"])\n", + "print(\"HPO terms:\", [(t.hpo_id, t.hpo_label) for t in vp_anatomy[\"hpo_terms\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "b52f739f", + "metadata": {}, + "source": [ + "\"Cerebellar hypoplasia\" - unlike Apple Sally's Dandy-Walker, this one *is* recognized correctly. Now the full build:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "d3023e9b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.102480Z", + "iopub.status.busy": "2026-07-28T13:28:17.102445Z", + "iopub.status.idle": "2026-07-28T13:28:17.143277Z", + "shell.execute_reply": "2026-07-28T13:28:17.142853Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=175.0mm, percentile=50.0%, ga=None, method=None\n" + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 25 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 25 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A|||||||||20260115140000...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|205^205.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 205^205.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 205.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 205.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=205.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&f...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|45^45%|%&percent^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|NM|SkullFetus.BiParietalDiameter^BPD|Fetus1|53^53.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.BiParietalDiameter^BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 53^53.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 53.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=value, value={'value': 53.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=53.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|NM|SkullFetus.VP_BiParietalDiameter_GA|Fetus1|149^21w 2d|d&weeks_days^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 149^21w 2d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.VP_BiParietalDiameter_Percentile|Fetus1|40^40%|%&percent^fmt...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_BiParietalDiameter_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 40^40%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BiParietalDiameter, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: BPD (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=BPD, field=percentile, value=40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=40.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|AbdomenFetus.AbdominalCircumference^AC|Fetus1|168^168.0|mm&millimeters^...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.AbdominalCircumference^AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 168^168.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 168.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=value, value={'value': 168.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=168.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|NM|AbdomenFetus.VP_AbdominalCircumference_GA|Fetus1|151^21w 4d|d&weeks_day...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 151^21w 4d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|NM|AbdomenFetus.VP_AbdominalCircumference_Percentile|Fetus1|38^38%|%&perc...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: AbdomenFetus.VP_AbdominalCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 38^38%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=AbdominalCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: AC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=AC, field=percentile, value=38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=38.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[10] Processing: OBX|11|NM|ExtremitiesFetus.FemurLength^FL|Fetus1|34^34.0|mm&millimeters^mm&milli...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.FemurLength^FL\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 34^34.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 34.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=value, value={'value': 34.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=34.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[11] Processing: OBX|12|NM|ExtremitiesFetus.VP_FemurLength_GA|Fetus1|150^21w 3d|d&weeks_days^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_GA\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 150^21w 3d\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=ga\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (ga)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: \n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=ga, value=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored ga=\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[12] Processing: OBX|13|NM|ExtremitiesFetus.VP_FemurLength_Percentile|Fetus1|42^42%|%&percent^fmt...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ExtremitiesFetus.VP_FemurLength_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 42^42%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FemurLength, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: Femur (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=Femur, field=percentile, value=42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=42.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[13] Processing: OBX|14|ST|BrainFetus.BrainAppearance^Brain appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.BrainAppearance^Brain appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BrainAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BrainAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.BrainAppearance^Brain appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[14] Processing: OBX|15|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal|...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[15] Processing: OBX|16|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypo...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[16] Processing: OBX|17|ST|FaceFetus.FaceAppearance^Face appearance|Fetus1|normal|||||||||2026011...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: FaceFetus.FaceAppearance^Face appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=FaceAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: FaceAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: FaceFetus.FaceAppearance^Face appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[17] Processing: OBX|18|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[18] Processing: OBX|19|ST|GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract ap...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=GastrointestinalTractAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: GastrointestinalTractAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: GastrointestinalTractFetus.GastrointestinalTractAppearance^GI tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[19] Processing: OBX|20|ST|SpineFetus.SpineAppearance^Spine appearance|Fetus1|normal|||||||||2026...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SpineFetus.SpineAppearance^Spine appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=SpineAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: SpineAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: SpineFetus.SpineAppearance^Spine appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[20] Processing: OBX|21|ST|UrinaryTractFetus.BladderAppearance^Bladder appearance|Fetus1|normal||...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.BladderAppearance^Bladder appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=BladderAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: BladderAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.BladderAppearance^Bladder appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[21] Processing: OBX|22|ST|UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearanc...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=UrogenitalTractAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: UrogenitalTractAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.UrogenitalTractAppearance^Urogenital tract appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[22] Processing: OBX|23|ST|UrinaryTractFetus.KidneyLAppearance^Left kidney appearance|Fetus1|norm...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyLAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyLAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyLAppearance^Left kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[23] Processing: OBX|24|ST|UrinaryTractFetus.KidneyRAppearance^Right kidney appearance|Fetus1|nor...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: normal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=KidneyRAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: KidneyRAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: UrinaryTractFetus.KidneyRAppearance^Right kidney appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[24] Processing: OBX|25|ST|RequestedProcedure.Indication^Indication^Order indication|1|Full secon...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: RequestedProcedure.Indication^Indication^Order indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: 1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: Full second trimester survey; anomaly follow-up\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Indication, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: RequestedProcedure.Indication^Indication^Order indication\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 4 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 205.0, 'unit': 'mm', 'percentile': 45.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 205.0 mm -> 205.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=205.0mm, percentile=45.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing BPD: {'value': 53.0, 'unit': 'mm', 'percentile': 40.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 53.0 mm -> 53.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=BPD, value=53.0mm, percentile=40.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for BPD\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing AC: {'value': 168.0, 'unit': 'mm', 'percentile': 38.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 168.0 mm -> 168.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=AC, value=168.0mm, percentile=38.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0034207 - Abnormal fetal gastrointestinal system morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0034207 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for AC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing Femur: {'value': 34.0, 'unit': 'mm', 'percentile': 42.0, 'ga': , 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 34.0 mm -> 34.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=Femur, value=34.0mm, percentile=42.0%, ga=, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0002823 - Abnormal femur morphology\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0002823 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for Femur\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 4 TermBins total\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n" + "id: full00099-fetus-1\n", + " HP:0000240 Abnormality of skull size excluded=True | Biometry: HC: 205.0 mm (45.0%) at 21w3d\n", + " HP:0034207 Abnormal fetal gastrointestinal system morphology excluded=True | Biometry: AC: 168.0 mm (38.0%) at 21w4d\n", + " HP:0002823 Abnormal femur morphology excluded=True | Biometry: Femur: 34.0 mm (42.0%) at 21w3d\n", + " HP:0001321 Cerebellar hypoplasia excluded=False | Fetal anatomy\n" ] - }, + } + ], + "source": [ + "from prenatalppkt.builders import build_viewpoint_phenopacket\n", + "\n", + "vp_pps = build_viewpoint_phenopacket(vp_data, hpo_parser, now_ts, accession_id=\"FULL00099\")\n", + "vp_pp = vp_pps[0]\n", + "print(\"id:\", vp_pp.id)\n", + "for pf in vp_pp.phenotypic_features:\n", + " print(\" \", pf.type.id, pf.type.label, \"excluded=\"+str(pf.excluded), \"|\", pf.description)" + ] + }, + { + "cell_type": "markdown", + "id": "550e2c41", + "metadata": {}, + "source": [ + "Only 4 features, not 5 - BPD is missing even though the extractor found it. HC and BPD both map to the same HPO term (`HP:0000240`, \"Abnormality of skull size\"), and the builder's HPO-id dedup keeps only the first occurrence. Expected, not a bug - the same dedup Apple Sally's Observer builder uses too.\n", + "\n", + "### The twin-anatomy bug, ViewPoint's independent copy\n", + "\n", + "ViewPoint HL7 had the exact same bug as Observer, found separately: `_parse_viewpoint_hl7_anatomy`'s own code comment claimed anatomy OBX segments \"aren't per-fetus tagged the same way biometry's are\" - checked directly against a real fixture, and that claim was wrong. Anatomy segments carry the same `Fetus1`/`Fetus2` sub-id biometry segments do." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "2959c6ca", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.176494Z", + "iopub.status.busy": "2026-07-28T13:28:17.176418Z", + "iopub.status.idle": "2026-07-28T13:28:17.206339Z", + "shell.execute_reply": "2026-07-28T13:28:17.205964Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n" + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 10 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 10 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 175^175.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 175.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 175.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=175.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 50^50%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumAppearance^Cerebellum appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypop...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: cerebellar hypoplasia\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=CerebellumDetails, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: CerebellumDetails\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.CerebellumDetails^Cerebellum details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[5] Processing: OBX|6|ST|Fetus.Identifier^Fetus Identifier|Fetus2|B...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: B\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[6] Processing: OBX|7|NM|SkullFetus.HeadCircumference^HC|Fetus2|170^170.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 170^170.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 170.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus2, measurement=HC, field=value, value={'value': 170.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=170.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[7] Processing: OBX|8|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus2|45^45%|%&percent^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 45^45%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus2, measurement=HC, field=percentile, value=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=45.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[8] Processing: OBX|9|ST|BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearanc...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=LateralVentricleLAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: LateralVentricleLAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[9] Processing: OBX|10|ST|BrainFetus.LateralVentricleLDetails^Left lateral ventricle details|Fet...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: BrainFetus.LateralVentricleLDetails^Left lateral ventricle details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus2\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: ventriculomegaly\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=LateralVentricleLDetails, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: LateralVentricleLDetails\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: BrainFetus.LateralVentricleLDetails^Left lateral ventricle details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 2 fetus records\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus2: 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 170.0, 'unit': 'mm', 'percentile': 45.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 175.0 mm -> 175.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=175.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 1 TermBins total\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 170.0, 'unit': 'mm', 'percentile': 45.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 170.0 mm -> 170.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=170.0mm, percentile=45.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 1 TermBins total\n" ] }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n" + "twin-fetus-1: {'HP:0001321', 'HP:0000240'}\n", + "twin-fetus-2: {'HP:0002119', 'HP:0000240'}\n" ] - }, + } + ], + "source": [ + "twin_hl7 = r\"\"\"\n", + "MSH|^~\\&|ViewPoint|Hospital|||20260115120000||ORU^R01|123456|P|2.4\n", + "OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A\n", + "OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&millimeters\n", + "OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&formatted\n", + "OBX|4|ST|BrainFetus.CerebellumAppearance^Cerebellum appearance|Fetus1|abnormal\n", + "OBX|5|ST|BrainFetus.CerebellumDetails^Cerebellum details|Fetus1|cerebellar hypoplasia\n", + "OBX|6|ST|Fetus.Identifier^Fetus Identifier|Fetus2|B\n", + "OBX|7|NM|SkullFetus.HeadCircumference^HC|Fetus2|170^170.0|mm&millimeters^mm&millimeters\n", + "OBX|8|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus2|45^45%|%&percent^fmt&formatted\n", + "OBX|9|ST|BrainFetus.LateralVentricleLAppearance^Left lateral ventricle appearance|Fetus2|abnormal\n", + "OBX|10|ST|BrainFetus.LateralVentricleLDetails^Left lateral ventricle details|Fetus2|ventriculomegaly\n", + "\"\"\"\n", + "twin_vp_pps = build_viewpoint_phenopacket(twin_hl7, hpo_parser, now_ts, accession_id=\"TWIN\")\n", + "for pp in twin_vp_pps:\n", + " hpo_ids = {pf.type.id for pf in pp.phenotypic_features}\n", + " print(f\"{pp.id}: {hpo_ids}\")" + ] + }, + { + "cell_type": "markdown", + "id": "63a11f95", + "metadata": {}, + "source": [ + "Fetus 1's cerebellar hypoplasia (`HP:0001321`) and fetus 2's ventriculomegaly (`HP:0002119`) each stay with their own fetus. Confirmed fixed, same pattern as the Observer fix.\n", + "\n", + "### A fenominal gap that needed its own synthetic message to prove\n", + "\n", + "The real ViewPoint anatomy fixture that has \"mild aortic arch narrowing\" has no biometry data at all, so it produces zero Phenopackets through the full builder - there was no way to prove the gap survives to a real final Phenopacket without building one by hand." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "84789133", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-28T13:28:17.217741Z", + "iopub.status.busy": "2026-07-28T13:28:17.217679Z", + "iopub.status.idle": "2026-07-28T13:28:17.240826Z", + "shell.execute_reply": "2026-07-28T13:28:17.240129Z" + } + }, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Loaded mappings for: ['head_circumference', 'biparietal_diameter', 'femur_length', 'abdominal_circumference', 'occipitofrontal_diameter', 'crown_rump_length', 'nuchal_translucency', 'tibia_length', 'fibula_length', 'radius_length', 'ulna_length', 'foot_length']\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Extracted 5 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing 5 OBX segments\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[0] Processing: OBX|1|ST|Fetus.Identifier^Fetus Identifier|Fetus1|A...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: A\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=Identifier, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: Fetus.Identifier^Fetus Identifier\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[1] Processing: OBX|2|NM|SkullFetus.HeadCircumference^HC|Fetus1|175^175.0|mm&millimeters^mm&mill...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.HeadCircumference^HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 175^175.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (value)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: {'value': 175.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=value, value={'value': 175.0, 'unit': 'mm'}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new fetus record: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Created new measurement: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored value=175.0 unit=mm\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[2] Processing: OBX|3|NM|SkullFetus.VP_HeadCircumference_Percentile|Fetus1|50^50%|%&percent^fmt&...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: SkullFetus.VP_HeadCircumference_Percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: 50^50%\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=HeadCircumference, type=percentile\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Mapped to standard name: HC\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Recognized: HC (percentile)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Extracted value: 50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Parsed: fetus=Fetus1, measurement=HC, field=percentile, value=50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Stored percentile=50.0\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[3] Processing: OBX|4|ST|ChestFetus.ChestAppearance^Chest appearance|Fetus1|abnormal...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: abnormal\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ChestAppearance, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ChestAppearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ChestAppearance^Chest appearance\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:[4] Processing: OBX|5|ST|ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details|F...\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code: ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus ID: Fetus1\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Value field: mild aortic arch narrowing\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Parsed code: base=ThoracicDescAortaDetails, type=value\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Base code not in mapping: ThoracicDescAortaDetails\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Code not recognized as measurement: ChestFetus.ThoracicDescAortaDetails^Thoracic descending aorta details\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: -> Skipped (not a measurement field)\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Final measurements dict has 1 fetus records\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: Fetus Fetus1: 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7: - HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Creating TermBins from 1 measurements\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Processing HC: {'value': 175.0, 'unit': 'mm', 'percentile': 50.0, 'ga': None, 'method': None}\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Converted 175.0 mm -> 175.0 mm\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Creating TermBin: name=HC, value=175.0mm, percentile=50.0%, ga=None, method=None\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Selected HPO: HP:0000240 - Abnormality of skull size\n", + "DEBUG:prenatalppkt.etl.term_bin_factory:Created TermBin: HP:0000240 - normal=True\n", + "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created TermBin for HC\n", "DEBUG:prenatalppkt.etl.extractors.viewpoint_hl7:Created 1 TermBins total\n" ] }, @@ -8161,14 +2341,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "id": "618ac1d6", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.442273Z", - "iopub.status.busy": "2026-07-27T16:39:45.442190Z", - "iopub.status.idle": "2026-07-27T16:39:45.444917Z", - "shell.execute_reply": "2026-07-27T16:39:45.444591Z" + "iopub.execute_input": "2026-07-28T13:28:17.245925Z", + "iopub.status.busy": "2026-07-28T13:28:17.245863Z", + "iopub.status.idle": "2026-07-28T13:28:17.248264Z", + "shell.execute_reply": "2026-07-28T13:28:17.247931Z" } }, "outputs": [ @@ -8214,14 +2394,14 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "id": "98c3eb1d", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.446118Z", - "iopub.status.busy": "2026-07-27T16:39:45.446055Z", - "iopub.status.idle": "2026-07-27T16:39:45.452618Z", - "shell.execute_reply": "2026-07-27T16:39:45.452395Z" + "iopub.execute_input": "2026-07-28T13:28:17.249184Z", + "iopub.status.busy": "2026-07-28T13:28:17.249098Z", + "iopub.status.idle": "2026-07-28T13:28:17.257683Z", + "shell.execute_reply": "2026-07-28T13:28:17.257442Z" } }, "outputs": [ @@ -8266,14 +2446,14 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "id": "f002f205", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:45.453663Z", - "iopub.status.busy": "2026-07-27T16:39:45.453537Z", - "iopub.status.idle": "2026-07-27T16:39:46.083641Z", - "shell.execute_reply": "2026-07-27T16:39:46.083262Z" + "iopub.execute_input": "2026-07-28T13:28:17.258555Z", + "iopub.status.busy": "2026-07-28T13:28:17.258515Z", + "iopub.status.idle": "2026-07-28T13:28:17.498809Z", + "shell.execute_reply": "2026-07-28T13:28:17.498383Z" } }, "outputs": [ @@ -8325,14 +2505,14 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "id": "746995d0", "metadata": { "execution": { - "iopub.execute_input": "2026-07-27T16:39:46.084603Z", - "iopub.status.busy": "2026-07-27T16:39:46.084539Z", - "iopub.status.idle": "2026-07-27T16:39:46.108414Z", - "shell.execute_reply": "2026-07-27T16:39:46.107998Z" + "iopub.execute_input": "2026-07-28T13:28:17.499811Z", + "iopub.status.busy": "2026-07-28T13:28:17.499763Z", + "iopub.status.idle": "2026-07-28T13:28:17.519161Z", + "shell.execute_reply": "2026-07-28T13:28:17.518829Z" } }, "outputs": [ @@ -8340,188 +2520,32 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:prenatalppkt.biometry_reference:Loaded measures for intergrowth: ['head_circumference', 'biparietal_diameter', 'abdominal_circumference', 'femur_length', 'occipitofrontal_diameter']\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Loading HPO mappings from /Users/jv2684/Desktop/Illini+Uni/Personalized_Genomic_Medicine-Precision_Genomics_Laboratory/PreGen/prenatalppkt/data/mappings/biometry_hpo_mappings.yaml\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for head_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for biparietal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for femur_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for abdominal_circumference\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for occipitofrontal_diameter\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for crown_rump_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for nuchal_translucency\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for tibia_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for fibula_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for radius_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for ulna_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:prenatalppkt.measurement_eval:Initialized mapper for foot_length\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ + "DEBUG:prenatalppkt.biometry_reference:Loaded measures for intergrowth: ['head_circumference', 'biparietal_diameter', 'abdominal_circumference', 'femur_length', 'occipitofrontal_diameter']\n", + "INFO:prenatalppkt.measurement_eval:Loading HPO mappings from /Users/jv2684/Desktop/Illini+Uni/Personalized_Genomic_Medicine-Precision_Genomics_Laboratory/PreGen/prenatalppkt/data/mappings/biometry_hpo_mappings.yaml\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for head_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for biparietal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for femur_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for abdominal_circumference\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for occipitofrontal_diameter\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for crown_rump_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for nuchal_translucency\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for tibia_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for fibula_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for radius_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for ulna_length\n", + "DEBUG:prenatalppkt.mapping_loader:Loaded 8 bins for foot_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for head_circumference\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for biparietal_diameter\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for femur_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for abdominal_circumference\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for occipitofrontal_diameter\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for crown_rump_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for nuchal_translucency\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for tibia_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for fibula_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for radius_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for ulna_length\n", + "INFO:prenatalppkt.measurement_eval:Initialized mapper for foot_length\n", "INFO:prenatalppkt.phenotypic_export:Initialized PhenotypicExporter with intergrowth reference\n" ] }, @@ -8569,7 +2593,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" },