diff --git a/Makefile b/Makefile index 80a7cb2..7894d06 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,51 @@ datamodel: poetry run gen-python kg_microbe_merge/schema/merge_schema.yaml > kg_microbe_merge/schema/merge_datamodel.py -kg-microbe-core: +kg-microbe-core: datamodel poetry run kg merge -m duckdb -s "bacdive, mediadive, madin_etal, rhea_mappings, bactotraits, chebi, ec, envo, go, ncbitaxon, upa" --merge-label $@ + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py $@ + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/$@/$@_missing_nodes_with_category.tsv ]; then \ + cd data/merged/$@ && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv $@_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm $@_missing_nodes.tsv $@_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi -kg-microbe-function: +kg-microbe-function: datamodel poetry run kg merge -m duckdb -s "bacdive, mediadive, madin_etal, rhea_mappings, bactotraits, chebi, ec, envo, go, ncbitaxon, upa, uniprot_functional_microbes" --merge-label $@ + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py $@ + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/$@/$@_missing_nodes_with_category.tsv ]; then \ + cd data/merged/$@ && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv $@_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm $@_missing_nodes.tsv $@_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi -kg-microbe-biomedical: +kg-microbe-biomedical: datamodel poetry run kg merge -m duckdb -s "bacdive, mediadive, madin_etal, rhea_mappings, bactotraits, chebi, ec, envo, go, ncbitaxon, upa, hp, mondo, disbiome, ctd, wallen_etal, uniprot_human" --merge-label $@ + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py $@ + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/$@/$@_missing_nodes_with_category.tsv ]; then \ + cd data/merged/$@ && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv $@_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm $@_missing_nodes.tsv $@_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi -kg-microbe-biomedical-function: +kg-microbe-biomedical-function: datamodel poetry run kg merge -m duckdb --merge-label $@ + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py $@ + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/$@/$@_missing_nodes_with_category.tsv ]; then \ + cd data/merged/$@ && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv $@_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm $@_missing_nodes.tsv $@_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi -kg-microbe-function-cat: +kg-microbe-function-cat: kg-microbe-core cd data/raw/uniprot_functional_microbes && \ grep UniprotKB: nodes.tsv > nodes_UniprotKB.tsv && \ tail -n +2 edges.tsv | cut -f1,2,3 > edges_data_clean.tsv && \ @@ -22,19 +54,21 @@ kg-microbe-function-cat: mkdir -p kg-microbe-function-cat && \ cd kg-microbe-core && \ tail -n +2 merged-kg_edges.tsv > edges_data.tsv && \ + head -1 merged-kg_edges.tsv > edges_header.tsv && \ cd ../ && \ cd kg-microbe-function-cat && \ - cat ../kg-microbe-core/merged-kg_nodes.tsv ../../raw/uniprot_functional_microbes/nodes_UniprotKB.tsv ../kg-microbe-core/kg-microbe-core_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ - cat ../kg-microbe-core/edges_header.tsv ../kg-microbe-core/edges_data.tsv ../../raw/uniprot_functional_microbes/edges_data_clean.tsv > merged-kg_edges.tsv && \ - cd ../../../ && \ - poetry run python kg_microbe_merge/utils/edge_vs_node_check.py kg-microbe-function-cat && \ - cd data/merged/kg-microbe-function-cat && \ - mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ - cat merged-kg_nodes_part.tsv kg-microbe-function-cat_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ - cd ../../../ - -kg-microbe-biomedical-function-cat: - + cat ../kg-microbe-core/merged-kg_nodes.tsv ../../raw/uniprot_functional_microbes/nodes_UniprotKB.tsv > merged-kg_nodes.tsv && \ + cat ../kg-microbe-core/edges_header.tsv ../kg-microbe-core/edges_data.tsv ../../raw/uniprot_functional_microbes/edges_data_clean.tsv > merged-kg_edges.tsv + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py kg-microbe-function-cat + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/kg-microbe-function-cat/kg-microbe-function-cat_missing_nodes_with_category.tsv ]; then \ + cd data/merged/kg-microbe-function-cat && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv kg-microbe-function-cat_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm kg-microbe-function-cat_missing_nodes.tsv kg-microbe-function-cat_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi + +kg-microbe-biomedical-function-cat: kg-microbe-biomedical cd data/raw/uniprot_functional_microbes && \ grep UniprotKB: nodes.tsv > nodes_UniprotKB.tsv && \ tail -n +2 edges.tsv | cut -f1,2,3 > edges_data_clean.tsv && \ @@ -42,17 +76,43 @@ kg-microbe-biomedical-function-cat: cd ../../merged && \ cd kg-microbe-biomedical && \ tail -n +2 merged-kg_edges.tsv > edges_data.tsv && \ + head -1 merged-kg_edges.tsv > edges_header.tsv && \ cd ../ && \ mkdir -p kg-microbe-biomedical-function-cat && \ cd kg-microbe-biomedical-function-cat && \ - cat ../kg-microbe-biomedical/merged-kg_nodes.tsv ../../raw/uniprot_functional_microbes/nodes_UniprotKB.tsv > merged-kg_nodes.tsv ../kg-microbe-biomedical/kg-microbe-biomedical_missing_nodes_with_category.tsv && \ - cat ../kg-microbe-core/edges_header.tsv ../kg-microbe-biomedical/edges_data.tsv ../../raw/uniprot_functional_microbes/edges_data_clean.tsv > merged-kg_edges.tsv && \ - cd ../../../ && \ - poetry run python kg_microbe_merge/utils/edge_vs_node_check.py kg-microbe-biomedical-function-cat && \ - cd data/merged/kg-microbe-biomedical-function-cat && \ - mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ - cat merged-kg_nodes_part.tsv kg-microbe-biomedical-function-cat_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ - cd ../../../ + cat ../kg-microbe-biomedical/merged-kg_nodes.tsv ../../raw/uniprot_functional_microbes/nodes_UniprotKB.tsv > merged-kg_nodes.tsv && \ + cat ../kg-microbe-biomedical/edges_header.tsv ../kg-microbe-biomedical/edges_data.tsv ../../raw/uniprot_functional_microbes/edges_data_clean.tsv > merged-kg_edges.tsv + poetry run python kg_microbe_merge/utils/edge_vs_node_check.py kg-microbe-biomedical-function-cat + @# If missing nodes were found, add them to the nodes file + @if [ -f data/merged/kg-microbe-biomedical-function-cat/kg-microbe-biomedical-function-cat_missing_nodes_with_category.tsv ]; then \ + cd data/merged/kg-microbe-biomedical-function-cat && \ + mv merged-kg_nodes.tsv merged-kg_nodes_part.tsv && \ + cat merged-kg_nodes_part.tsv kg-microbe-biomedical-function-cat_missing_nodes_with_category.tsv > merged-kg_nodes.tsv && \ + rm kg-microbe-biomedical-function-cat_missing_nodes.tsv kg-microbe-biomedical-function-cat_missing_nodes_with_category.tsv merged-kg_nodes_part.tsv; \ + fi + +clean: + # Remove generated datamodel + rm -f kg_microbe_merge/schema/merge_datamodel.py + + # Remove all merged directories + rm -rf data/merged/kg-microbe-core + rm -rf data/merged/kg-microbe-function + rm -rf data/merged/kg-microbe-function-cat + rm -rf data/merged/kg-microbe-biomedical + rm -rf data/merged/kg-microbe-biomedical-function + rm -rf data/merged/kg-microbe-biomedical-function-cat + + # Remove temporary files created during concatenation + rm -f data/raw/uniprot_functional_microbes/nodes_UniprotKB.tsv + rm -f data/raw/uniprot_functional_microbes/edges_data_clean.tsv + rm -f data/raw/uniprot_functional_microbes/edges_header_clean.tsv + + # Remove any edge_data and edge_header files in merged directories + find data/merged -name "edges_data.tsv" -type f -delete 2>/dev/null || true + find data/merged -name "edges_header.tsv" -type f -delete 2>/dev/null || true + + @echo "Cleaned all generated files" include kg-microbe-merge.Makefile diff --git a/hpc/run_edge_vs_node_check.sl b/hpc/run_edge_vs_node_check.sl new file mode 100644 index 0000000..f439c15 --- /dev/null +++ b/hpc/run_edge_vs_node_check.sl @@ -0,0 +1,20 @@ +#!/bin/bash +#SBATCH --account=m4689 +#SBATCH --qos=regular +#SBATCH --constraint=cpu +#SBATCH --time=360 +#SBATCH --mem=470GB +#SBATCH --ntasks=1 +#SBATCH --output=%j.out +#SBATCH --error=%j.err +#SBATCH -N 1 +#SBATCH --mail-type=BEGIN,END +#SBATCH --mail-user=MJoachimiak@lbl.gov + +module load python/3.10 + +cd /global/cfs/cdirs/m4689/master/kg-microbe-merge +source venv-merge/bin/activate + +time poetry run python kg_microbe_merge/utils/edge_vs_node_check.py + diff --git a/kg_microbe_merge/run.py b/kg_microbe_merge/run.py index 0e6eafb..68b250c 100644 --- a/kg_microbe_merge/run.py +++ b/kg_microbe_merge/run.py @@ -139,11 +139,11 @@ def merge( merge_kg_object.merged_graph = merged_graph_object if merge_tool == "duckdb": if merge_label: - merged_nodes_output_path = MERGED_DATA_DIR / merge_label / "nodes.tsv" - merged_edges_output_path = MERGED_DATA_DIR / merge_label / "edges.tsv" + merged_nodes_output_path = MERGED_DATA_DIR / merge_label / f"merged-kg_nodes.tsv" + merged_edges_output_path = MERGED_DATA_DIR / merge_label / f"merged-kg_edges.tsv" else: - merged_nodes_output_path = MERGED_DATA_DIR / "nodes.tsv" - merged_edges_output_path = MERGED_DATA_DIR / "edges.tsv" + merged_nodes_output_path = MERGED_DATA_DIR / "merged-kg_nodes.tsv" + merged_edges_output_path = MERGED_DATA_DIR / "merged-kg_edges.tsv" duckdb_merge( node_paths, diff --git a/kg_microbe_merge/utils/edge_vs_node_check.py b/kg_microbe_merge/utils/edge_vs_node_check.py index 91e1833..34e3eac 100644 --- a/kg_microbe_merge/utils/edge_vs_node_check.py +++ b/kg_microbe_merge/utils/edge_vs_node_check.py @@ -26,13 +26,32 @@ def main(kg_path): con = duckdb.connect() # Load only the necessary columns from the TSV files into DuckDB tables - con.execute( - f""" - CREATE TABLE edges AS - SELECT subject AS subject_id, object AS object_id - FROM read_csv_auto('data/merged/{kg_path}/merged-kg_edges.tsv', delim='\t', null_padding=true) - """ - ) + # Check which columns are available in the edges file + edges_df = con.execute(f"SELECT * FROM read_csv_auto('data/merged/{kg_path}/merged-kg_edges.tsv', delim='\t', null_padding=true) LIMIT 0").df() + columns = list(edges_df.columns) + + # Handle different column configurations + if 'subject' in columns and 'object' in columns: + # Standard case with subject and object columns + con.execute( + f""" + CREATE TABLE edges AS + SELECT subject AS subject_id, object AS object_id + FROM read_csv_auto('data/merged/{kg_path}/merged-kg_edges.tsv', delim='\t', null_padding=true) + """ + ) + elif 'predicate' in columns and 'object' in columns and len(columns) == 2: + # Case for concatenated files where subject column is missing + # We'll only check objects in this case + con.execute( + f""" + CREATE TABLE edges AS + SELECT object AS object_id + FROM read_csv_auto('data/merged/{kg_path}/merged-kg_edges.tsv', delim='\t', null_padding=true) + """ + ) + else: + raise ValueError(f"Unexpected column structure in edges file: {columns}") con.execute( f""" CREATE TABLE nodes AS @@ -42,17 +61,30 @@ def main(kg_path): ) # Check whether all subject and object IDs are represented as node IDs - query = """ - WITH distinct_ids AS ( - SELECT DISTINCT subject_id AS id FROM edges - UNION - SELECT DISTINCT object_id AS id FROM edges - ) - SELECT distinct_ids.id - FROM distinct_ids - LEFT JOIN nodes ON distinct_ids.id = nodes.id - WHERE nodes.id IS NULL - """ + if 'subject' in columns and 'object' in columns: + # Standard case - check both subject and object IDs + query = """ + WITH distinct_ids AS ( + SELECT DISTINCT subject_id AS id FROM edges + UNION + SELECT DISTINCT object_id AS id FROM edges + ) + SELECT distinct_ids.id + FROM distinct_ids + LEFT JOIN nodes ON distinct_ids.id = nodes.id + WHERE nodes.id IS NULL + """ + else: + # Concatenated case - only check object IDs + query = """ + WITH distinct_ids AS ( + SELECT DISTINCT object_id AS id FROM edges + ) + SELECT distinct_ids.id + FROM distinct_ids + LEFT JOIN nodes ON distinct_ids.id = nodes.id + WHERE nodes.id IS NULL + """ # Execute the query and fetch all results missing_ids = con.execute(query).fetchall()