diff --git a/CHANGES.rst b/CHANGES.rst index 2b250f4e59..e06eec67b2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -25,7 +25,6 @@ esa.emds.einsteinprobe - New module to access the ESA Einstein Probe Science Archive. [#3511] - API changes ----------- @@ -86,6 +85,11 @@ vo_conesearch Service fixes and enhancements ------------------------------ +esa.jwst +^^^^^^^^^^^^^^ + +- Refactored to use PyVo instead of deprecated TapPlus class [#3592] + esa.xmm_newton ^^^^^^^^^^^^^^ diff --git a/astroquery/esa/jwst/__init__.py b/astroquery/esa/jwst/__init__.py index 39dbbb0564..a0d7b67cc8 100644 --- a/astroquery/esa/jwst/__init__.py +++ b/astroquery/esa/jwst/__init__.py @@ -11,6 +11,11 @@ from astropy import config as _config +from astropy.config import paths +import os + +JWST_COMMON_SERVER = "https://jwst.esac.esa.int/server/" +JWST_TAP_COMMON = "tap" class Conf(_config.ConfigNamespace): @@ -18,39 +23,81 @@ class Conf(_config.ConfigNamespace): Configuration parameters for `astroquery.esa.jwst`. """ - JWST_TAP_SERVER = _config.ConfigItem("https://jwst.esac.esa.int/server/tap", "eJWST TAP Server") - JWST_DATA_SERVER = _config.ConfigItem("https://jwst.esac.esa.int/server/data?", "eJWST Data Server") - JWST_TOKEN = _config.ConfigItem("jwstToken", "eJWST token") - JWST_MESSAGES = _config.ConfigItem("notification?action=GetNotifications", "eJWST Messages") + JWST_DOMAIN_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER, "eJWST TAP Common Server")) + + JWST_TAP_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER + JWST_TAP_COMMON, + "eJWST TAP Server")) + + JWST_DATA_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER + 'data?', + "eJWST Data Server")) + + JWST_TABLES_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER + JWST_TAP_COMMON + "/tables", + "eJWST TAP Common Server")) + + JWST_TARGET_ACTION = ( + _config.ConfigItem("servlet/target-resolver?", + "eJWST Target Resolver")) + + JWST_MESSAGES = ( + _config.ConfigItem("notification?action=GetNotifications", + "JWST Messages")) + + JWST_UPLOAD = ( + _config.ConfigItem(JWST_COMMON_SERVER + "Upload", + "eJWST Upload Server")) + + TIMEOUT = 60 + + cache_location = os.path.join(paths.get_cache_dir(), 'astroquery/jwst',) + + JWST_LOGIN_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER + 'login', "eJWST Login Server")) - JWST_MAIN_TABLE = _config.ConfigItem("jwst.main", "JWST main table, combination of observation and plane tables.") + JWST_LOGOUT_SERVER = ( + _config.ConfigItem(JWST_COMMON_SERVER + 'logout', + "eJWST Logout Server")) - JWST_MAIN_TABLE_RA = _config.ConfigItem("target_ra", "Name of RA parameter in table") + JWST_TOKEN = ( + _config.ConfigItem("jwstToken", "eJWST token")) - JWST_MAIN_TABLE_DEC = _config.ConfigItem("target_dec", "Name of Dec parameter in table") + JWST_MAIN_TABLE = ( + _config.ConfigItem("jwst.archive", + "JWST main table, combination " + "of observation and plane tables.")) - JWST_ARTIFACT_TABLE = _config.ConfigItem("jwst.artifact", "JWST artifacts (data files) table.") + JWST_ARTIFACT_TABLE = ( + _config.ConfigItem("jwst.artifact", + "JWST artifacts (data files) table.")) - JWST_OBSERVATION_TABLE = _config.ConfigItem("jwst.observation", "JWST observation table") + JWST_OBSERVATION_TABLE = ( + _config.ConfigItem("jwst.observation", "JWST observation table")) - JWST_PLANE_TABLE = _config.ConfigItem("jwst.plane", "JWST plane table") + JWST_PLANE_TABLE = ( + _config.ConfigItem("jwst.plane", + "JWST plane table")) - JWST_OBS_MEMBER_TABLE = _config.ConfigItem("jwst.observationmember", "JWST observation member table") + JWST_OBS_MEMBER_TABLE = ( + _config.ConfigItem("jwst.observationmember", + "JWST observation member table")) - JWST_OBSERVATION_TABLE_RA = _config.ConfigItem("targetposition_coordinates_cval1", - "Name of RA parameter " - "in table") + JWST_OBSERVATION_TABLE_RA = ( + _config.ConfigItem("targetposition_coordinates_cval1", + "Name of RA parameter in table")) - JWST_OBSERVATION_TABLE_DEC = _config.ConfigItem("targetposition_coordinates_cval2", - "Name of Dec parameter " - "in table") + JWST_OBSERVATION_TABLE_DEC = ( + _config.ConfigItem("targetposition_coordinates_cval2", + "Name of Dec parameter in table")) - JWST_ARCHIVE_TABLE = _config.ConfigItem("jwst.archive", "JWST archive table") + JWST_ARCHIVE_TABLE = ( + _config.ConfigItem("jwst.archive", "JWST archive table")) conf = Conf() from .core import Jwst, JwstClass -from .data_access import JwstDataHandler -__all__ = ['Jwst', 'JwstClass', 'JwstDataHandler', 'Conf', 'conf'] +__all__ = ['Jwst', 'JwstClass', 'Conf', 'conf'] diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index b9347e5f4d..f9d5e3bffc 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -14,9 +14,11 @@ import os import shutil import tarfile as esatar +import tempfile import zipfile from datetime import datetime, timezone -from urllib.parse import urlencode + +from astropy.io.votable import from_table, writeto import astroquery.esa.utils.utils as esautils @@ -27,27 +29,27 @@ from astropy.units import Quantity from requests.exceptions import ConnectionError +from astroquery.esa.utils import EsaTap from astroquery.exceptions import RemoteServiceError from astroquery.ipac.ned import Ned -from astroquery.query import BaseQuery from astroquery.simbad import Simbad from astroquery.utils import commons -from astroquery.utils.tap import TapPlus from astroquery.vizier import Vizier from . import conf -from .data_access import JwstDataHandler __all__ = ['Jwst', 'JwstClass'] -# We do trust the ESA tar files, this is to avoid the new to Python 3.12 deprecation warning +# We do trust the ESA tar files, this is to avoid the new to Python 3.12 +# deprecation warning: # https://docs.python.org/3.12/library/tarfile.html#tarfile-extraction-filter if hasattr(esatar, "fully_trusted_filter"): - esatar.TarFile.extraction_filter = staticmethod(esatar.fully_trusted_filter) + esatar.TarFile.extraction_filter = ( + staticmethod(esatar.fully_trusted_filter)) -class JwstClass(BaseQuery): +class JwstClass(EsaTap): """ Proxy class to default TapPlus object (pointing to JWST Archive) @@ -55,8 +57,8 @@ class JwstClass(BaseQuery): JWST_DEFAULT_COLUMNS = ['observationid', 'calibrationlevel', 'public', 'dataproducttype', 'instrument_name', - 'energy_bandpassname', 'target_name', 'target_ra', - 'target_dec', 'position_bounds_center', + 'energy_bandpassname', 'target_name', 'targetposition_coordinates_cval1', + 'targetposition_coordinates_cval2', 'position_bounds_spoly'] PLANE_DATAPRODUCT_TYPES = ['image', 'cube', 'measurements', 'spectrum'] @@ -67,64 +69,27 @@ class JwstClass(BaseQuery): CAL_LEVELS = ['ALL', 1, 2, 3, -1] REQUESTED_OBSERVATION_ID = "Missing required argument: 'observation_id'" - def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=True): - if tap_plus_handler is None: - self.__jwsttap = TapPlus(url=conf.JWST_TAP_SERVER, - data_context='data', client_id='ASTROQUERY') - else: - self.__jwsttap = tap_plus_handler - - if data_handler is None: - self.__jwstdata = JwstDataHandler( - base_url=conf.JWST_DATA_SERVER) - else: - self.__jwstdata = data_handler - + ESA_ARCHIVE_NAME = "ESA JWST" + TAP_URL = conf.JWST_TAP_SERVER + LOGIN_URL = conf.JWST_LOGIN_SERVER + LOGOUT_URL = conf.JWST_LOGOUT_SERVER + UPLOAD_URL = conf.JWST_UPLOAD + + def __init__(self, + *, + show_messages=False, + auth_session=None, + tap_url=None): + super().__init__(auth_session=auth_session, tap_url=tap_url) if show_messages: self.get_status_messages() - def load_tables(self, *, only_names=False, include_shared_tables=False, - verbose=False): - """Loads all public tables - TAP & TAP+ - - Parameters - ---------- - only_names : bool, TAP+ only, optional, default 'False' - True to load table names only - include_shared_tables : bool, TAP+, optional, default 'False' - True to include shared tables - verbose : bool, optional, default 'False' - flag to display information about the process - - Returns - ------- - A list of table objects - """ - return self.__jwsttap.load_tables(only_names=only_names, - include_shared_tables=include_shared_tables, - verbose=verbose) - - def load_table(self, table, *, verbose=False): - """Loads the specified table - TAP+ only - - Parameters - ---------- - table : str, mandatory - full qualified table name (i.e. schema name + table name) - verbose : bool, optional, default 'False' - flag to display information about the process - - Returns - ------- - A table object - """ - return self.__jwsttap.load_table(table, verbose=verbose) - - def launch_job(self, query, *, name=None, output_file=None, - output_format="votable", verbose=False, dump_to_file=False, - background=False, upload_resource=None, upload_table_name=None, + def launch_job(self, query, *, + output_file=None, + output_format="votable", + verbose=False, + upload_resource=None, + upload_table_name=None, async_job=False): """Launches a synchronous or asynchronous job TAP & TAP+ @@ -145,11 +110,6 @@ def launch_job(self, query, *, name=None, output_file=None, 'fits': str, FITS format verbose : bool, optional, default 'False' flag to display information about the process - dump_to_file : bool, optional, default 'False' - if True, the results are saved in a file instead of using memory - background : bool, optional, default 'False' - when the job is executed in asynchronous mode, this flag specifies - whether the execution will wait until results are available upload_resource: str, optional, default None resource to be uploaded to UPLOAD_SCHEMA upload_table_name: str, required if uploadResource is provided @@ -162,83 +122,98 @@ def launch_job(self, query, *, name=None, output_file=None, ------- A Job object """ - if async_job: - return (self.__jwsttap.launch_job_async(query=query, - name=name, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file, - background=background, - upload_resource=upload_resource, - upload_table_name=upload_table_name)) - else: - return self.__jwsttap.launch_job(query=query, - name=name, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file, - upload_resource=upload_resource, - upload_table_name=upload_table_name) - - def load_async_job(self, *, jobid=None, name=None, verbose=False): - """Loads an asynchronous job - TAP & TAP+ - Parameters - ---------- - jobid : str, mandatory if no name is provided, default None - job identifier - name : str, mandatory if no jobid is provided, default None - job name - verbose : bool, optional, default 'False' - flag to display information about the process + if query == "" and upload_resource is None: + raise ValueError( + "Either 'query' or 'upload_resource' must be provided." + ) - Returns - ------- - A Job object - """ - return self.__jwsttap.load_async_job(jobid=jobid, name=name, verbose=verbose) + if query and upload_resource is not None: + raise ValueError( + "Cannot define both 'query' and " + "'upload_resource' simultaneously." + ) - def search_async_jobs(self, *, jobfilter=None, verbose=False): - """Searches for jobs applying the specified filter - TAP+ only + if query and upload_resource is None and upload_table_name is None: + if verbose: + print("Executing query (no upload requested)...") + result = self.query_tap( + query, + async_job=async_job, + output_file=output_file, + output_format=output_format, + verbose=verbose, + ) + return result + + if query and upload_resource is None and upload_table_name is not None: - Parameters - ---------- - jobfilter : JobFilter, optional, default None - job filter - verbose : bool, optional, default 'False' - flag to display information about the process + if verbose: + print("Executing query and preparing result upload...") - Returns - ------- - A list of Job objects - """ - return self.__jwsttap.search_async_jobs(jobfilter=jobfilter, verbose=verbose) + result = self.query_tap( + query, + async_job=async_job, + output_file=output_file, + output_format=output_format, + verbose=verbose, + ) - def list_async_jobs(self, *, verbose=False): - """Returns all the asynchronous jobs - TAP & TAP+ + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".vot") + temp_path = tmp.name + tmp.close() - Parameters - ---------- - verbose : bool, optional, default 'False' - flag to display information about the process + try: + votable = from_table(result) - Returns - ------- - A list of Job objects - """ - return self.__jwsttap.list_async_jobs(verbose=verbose) + # change bit to boolean for public and other columns + self._fix_jwst_boolean_fields(votable) + writeto(votable, temp_path) + + if verbose: + print(f"Temporary VOTable written: {temp_path}") + + self.upload_table( + conf.JWST_UPLOAD, + upload_resource=temp_path, + table_name=upload_table_name, + verbose=verbose + ) + + finally: + if os.path.exists(temp_path): + os.remove(temp_path) + if verbose: + print(f"Deleted temporary VOTable: {temp_path}") + + return result + + if query == "" and upload_resource is not None: + if upload_table_name is None: + raise ValueError( + "upload_table_name must be provided when " + "upload_resource is used." + ) + + if verbose: + print("Uploading user-provided file...") + + self.upload_table( + conf.JWST_UPLOAD, + upload_resource=upload_resource, + table_name=upload_table_name, + verbose=verbose + ) + + return None + return None def query_region(self, coordinate, *, radius=None, width=None, height=None, observation_id=None, - cal_level="Top", + cal_level=None, prod_type=None, instrument_name=None, filter_name=None, @@ -262,10 +237,9 @@ def query_region(self, coordinate, *, box height observation_id : str, optional, default None get the observation given by its ID. - cal_level : object, optional, default 'Top' + cal_level : integer, optional, default None get the planes with the given calibration level. Options are: - 'Top': str, only the planes with the highest calibration level - 1,2,3: int, the given calibration level + 1,2,3 prod_type : str, optional, default None get the observations providing the given product type. Options are: 'image','cube','measurements','spectrum': str, only results of the @@ -295,49 +269,57 @@ def query_region(self, coordinate, *, The job results (astropy.table). """ coord = self.__get_coord_input(value=coordinate, msg="coordinate") - job = None if radius is not None: - job = self.cone_search(coordinate=coord, - radius=radius, - only_public=only_public, - observation_id=observation_id, - cal_level=cal_level, - prod_type=prod_type, - instrument_name=instrument_name, - filter_name=filter_name, - proposal_id=proposal_id, - show_all_columns=show_all_columns, - async_job=async_job, verbose=verbose) + return self.cone_search(coordinate=coord, + radius=radius, + only_public=only_public, + observation_id=observation_id, + cal_level=cal_level, + prod_type=prod_type, + instrument_name=instrument_name, + filter_name=filter_name, + proposal_id=proposal_id, + show_all_columns=show_all_columns, + async_job=async_job, verbose=verbose) else: raHours, dec = commons.coord_to_radec(coord) ra = raHours * 15.0 # Converts to degrees - widthQuantity = self.__get_quantity_input(value=width, msg="width") - heightQuantity = self.__get_quantity_input(value=height, msg="height") + widthQuantity = self.__get_quantity_input( + value=width, msg="width") + heightQuantity = self.__get_quantity_input( + value=height, msg="height") widthDeg = widthQuantity.to(units.deg) heightDeg = heightQuantity.to(units.deg) - obsid_cond = self.__get_observationid_condition(value=observation_id) - cal_level_condition = self.__get_callevel_condition(cal_level=cal_level) - public_condition = self.__get_public_condition(only_public=only_public) - prod_cond = self.__get_plane_dataproducttype_condition(prod_type=prod_type) - instr_cond = self.__get_instrument_name_condition(value=instrument_name) - filter_name_cond = self.__get_filter_name_condition(value=filter_name) - props_id_cond = self.__get_proposal_id_condition(value=proposal_id) + obsid_cond = self.__get_observationid_condition( + value=observation_id) + cal_level_condition = self.__get_callevel_condition( + cal_level=cal_level) + public_condition = self.__get_public_condition( + only_public=only_public) + prod_cond = self.__get_plane_dataproducttype_condition( + prod_type=prod_type) + instr_cond = self.__get_instrument_name_condition( + value=instrument_name) + filter_name_cond = self.__get_filter_name_condition( + value=filter_name) + props_id_cond = self.__get_proposal_id_condition( + value=proposal_id) columns = str(', '.join(self.JWST_DEFAULT_COLUMNS)) if show_all_columns: columns = '*' query = (f"SELECT DISTANCE(POINT('ICRS'," - f"{str(conf.JWST_MAIN_TABLE_RA)}," - f"{str(conf.JWST_MAIN_TABLE_DEC)} ), " + f"{str(conf.JWST_OBSERVATION_TABLE_RA)}," + f"{str(conf.JWST_OBSERVATION_TABLE_DEC)} ), " f"POINT('ICRS',{str(ra)},{str(dec)} )) " f"AS dist, {columns} " - f"FROM {str(conf.JWST_MAIN_TABLE)} " + f"FROM {str(conf.JWST_ARCHIVE_TABLE)} " f"WHERE CONTAINS(" f"POINT('ICRS'," - f"{str(conf.JWST_MAIN_TABLE_RA)}," - f"{str(conf.JWST_MAIN_TABLE_DEC)})," + f"{str(conf.JWST_OBSERVATION_TABLE_RA)}," + f"{str(conf.JWST_OBSERVATION_TABLE_DEC)})," f"BOX('ICRS',{str(ra)},{str(dec)}, " f"{str(widthDeg.value)}, " f"{str(heightDeg.value)}))=1 " @@ -348,18 +330,16 @@ def query_region(self, coordinate, *, f"{instr_cond}" f"{filter_name_cond}" f"{props_id_cond}" - f"ORDER BY dist ASC") + f" ORDER BY dist ASC") if verbose: print(query) - if async_job: - job = self.__jwsttap.launch_job_async(query=query, verbose=verbose) - else: - job = self.__jwsttap.launch_job(query=query, verbose=verbose) - return job.get_results() + return self.query_tap(query=query, + async_job=async_job, + verbose=verbose) def cone_search(self, coordinate, radius, *, observation_id=None, - cal_level="Top", + cal_level=None, prod_type=None, instrument_name=None, filter_name=None, @@ -370,8 +350,7 @@ def cone_search(self, coordinate, radius, *, background=False, output_file=None, output_format="votable", - verbose=False, - dump_to_file=False): + verbose=False): """Cone search sorted by distance in sync/async mode TAP & TAP+ @@ -383,10 +362,9 @@ def cone_search(self, coordinate, radius, *, radius observation_id : str, optional, default None get the observation given by its ID. - cal_level : object, optional, default 'Top' + cal_level : int, optional, default None get the planes with the given calibration level. Options are: - 'Top': str, only the planes with the highest calibration level - 1,2,3: int, the given calibration level + 1,2,3 prod_type : str, optional, default None get the observations providing the given product type. Options are: 'image','cube','measurements','spectrum': str, only results of @@ -421,70 +399,64 @@ def cone_search(self, coordinate, radius, *, 'fits': str, FITS format verbose : bool, optional, default 'False' flag to display information about the process - dump_to_file : bool, optional, default 'False' - if True, the results are saved in a file instead of using memory Returns ------- A Job object """ - coord = self.__get_coord_input(value=coordinate, msg="coordinate") - ra_hours, dec = commons.coord_to_radec(coord) - ra = ra_hours * 15.0 # Converts to degrees - - obsid_condition = self.__get_observationid_condition(value=observation_id) - cal_level_condition = self.__get_callevel_condition(cal_level=cal_level) - public_condition = self.__get_public_condition(only_public=only_public) - prod_type_cond = self.__get_plane_dataproducttype_condition(prod_type=prod_type) - inst_name_cond = self.__get_instrument_name_condition(value=instrument_name) - filter_name_condition = self.__get_filter_name_condition(value=filter_name) - proposal_id_condition = self.__get_proposal_id_condition(value=proposal_id) + parsed_coords = commons.parse_coordinates(coordinates=coordinate) + ra = parsed_coords.ra.degree + dec = parsed_coords.dec.degree + + obsid_condition = self.__get_observationid_condition( + value=observation_id) + cal_level_condition = self.__get_callevel_condition( + cal_level=cal_level) + public_condition = self.__get_public_condition( + only_public=only_public) + prod_type_cond = self.__get_plane_dataproducttype_condition( + prod_type=prod_type) + inst_name_cond = self.__get_instrument_name_condition( + value=instrument_name) + filter_name_condition = self.__get_filter_name_condition( + value=filter_name) + proposal_id_condition = self.__get_proposal_id_condition( + value=proposal_id) columns = str(', '.join(self.JWST_DEFAULT_COLUMNS)) if show_all_columns: columns = '*' if radius is not None: - radius_quantity = self.__get_quantity_input(value=radius, msg="radius") + radius_quantity = self.__get_quantity_input( + value=radius, msg="radius") radius_deg = Angle(radius_quantity).to_value(units.deg) - query = (f"SELECT DISTANCE(POINT('ICRS'," - f"{str(conf.JWST_MAIN_TABLE_RA)}," - f"{str(conf.JWST_MAIN_TABLE_DEC)}), " - f"POINT('ICRS',{str(ra)},{str(dec)})) AS dist, {columns} " - f"FROM {str(conf.JWST_MAIN_TABLE)} WHERE CONTAINS(" - f"POINT('ICRS',{str(conf.JWST_MAIN_TABLE_RA)}," - f"{str(conf.JWST_MAIN_TABLE_DEC)})," + query = (f"SELECT {columns} " + f"FROM {str(conf.JWST_ARCHIVE_TABLE)} WHERE " + f"(position_bounds_spoly IS NOT NULL AND INTERSECTS(" f"CIRCLE('ICRS',{str(ra)},{str(dec)}, " - f"{str(radius_deg)}))=1" + f"{str(radius_deg)}),position_bounds_spoly)=1)" f"{obsid_condition}" f"{cal_level_condition}" f"{public_condition}" f"{prod_type_cond}" f"{inst_name_cond}" f"{filter_name_condition}" - f"{proposal_id_condition}" - f"ORDER BY dist ASC") - if async_job: - return self.__jwsttap.launch_job_async(query=query, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file, - background=background) - else: - return self.__jwsttap.launch_job(query=query, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file) + f"{proposal_id_condition} " + f"ORDER BY jwst.archive.calibrationlevel DESC") + return self.query_tap(query=query, + async_job=async_job, + output_file=output_file, + output_format=output_format, + verbose=verbose) def query_target(self, target_name, *, target_resolver="ALL", radius=None, width=None, height=None, observation_id=None, - cal_level="Top", + cal_level=None, prod_type=None, instrument_name=None, filter_name=None, @@ -493,7 +465,8 @@ def query_target(self, target_name, *, target_resolver="ALL", show_all_columns=False, async_job=False, verbose=False): - """Searches for a specific target defined by its name and other parameters + """Searches for a specific target defined by + its name and other parameters TAP & TAP+ Parameters @@ -513,10 +486,9 @@ def query_target(self, target_name, *, target_resolver="ALL", box height observation_id : str, optional, default None get the observation given by its ID. - cal_level : object, optional, default 'Top' + cal_level : int, optional, default None get the planes with the given calibration level. Options are: - 'Top': str, only the planes with the highest calibration level - 1,2,3: int, the given calibration level + 1,2,3 prod_type : str, optional, default None get the observations providing the given product type. Options are: 'image','cube','measurements','spectrum': str, only results of the @@ -546,8 +518,9 @@ def query_target(self, target_name, *, target_resolver="ALL", ------- The job results (astropy.table). """ - coordinates = self.resolve_target_coordinates(target_name=target_name, - target_resolver=target_resolver) + coordinates = ( + self.resolve_target_coordinates(target_name=target_name, + target_resolver=target_resolver)) return self.query_region(coordinate=coordinates, radius=radius, width=width, @@ -603,73 +576,6 @@ def resolve_target_coordinates(self, target_name, target_resolver): raise ValueError(f"This target name cannot be determined with" f" this resolver: {target_resolver}") - def remove_jobs(self, jobs_list, *, verbose=False): - """Removes the specified jobs - TAP+ - - Parameters - ---------- - jobs_list : str, mandatory - jobs identifiers to be removed - verbose : bool, optional, default 'False' - flag to display information about the process - - """ - return self.__jwsttap.remove_jobs(jobs_list=jobs_list, verbose=verbose) - - def save_results(self, job, *, verbose=False): - """Saves job results - TAP & TAP+ - - Parameters - ---------- - job : Job, mandatory - job - verbose : bool, optional, default 'False' - flag to display information about the process - """ - return self.__jwsttap.save_results(job=job, verbose=verbose) - - def login(self, *, user=None, password=None, credentials_file=None, - token=None, verbose=False): - """Performs a login. - TAP+ only - User and password can be used or a file that contains user name and - password (2 lines: one for user name and the following one for the - password) - - Parameters - ---------- - user : str, mandatory if 'file' is not provided, default None - login name - password : str, mandatory if 'file' is not provided, default None - user password - credentials_file : str, mandatory if no 'user' & 'password' are - provided - file containing user and password in two lines - token: str, optional - MAST token to have access to propietary data - verbose : bool, optional, default 'False' - flag to display information about the process - """ - self.__jwsttap.login(user=user, - password=password, - credentials_file=credentials_file, - verbose=verbose) - if token: - self.set_token(token=token) - - def logout(self, *, verbose=False): - """Performs a logout - TAP+ only - - Parameters - ---------- - verbose : bool, optional, default 'False' - flag to display information about the process - """ - return self.__jwsttap.logout(verbose=verbose) - def set_token(self, token): """Links a MAST token to the logged user @@ -678,12 +584,16 @@ def set_token(self, token): token: str, mandatory MAST token to have access to propietary data """ - subContext = conf.JWST_TOKEN - data = urlencode({"token": token}) - connHandler = self.__jwsttap._TapPlus__getconnhandler() - response = connHandler.execute_secure(subcontext=subContext, data=data, verbose=True) + + response = esautils.execute_servlet_request( + tap=self.tap, + query_params={"token": token}, + url=conf.JWST_DOMAIN_SERVER + conf.JWST_TARGET_ACTION + ) + if response.status == 403: - print("ERROR: MAST tokens cannot be assigned or requested by anonymous users") + print("ERROR: MAST tokens cannot be assigned " + "or requested by anonymous users") elif response.status == 500: print("ERROR: Server error when setting the token") else: @@ -693,18 +603,25 @@ def get_status_messages(self): """Retrieve the messages to inform users about the status of JWST TAP """ - try: - subContext = conf.JWST_MESSAGES - connHandler = self.__jwsttap._TapPlus__getconnhandler() - response = connHandler.execute_tapget(subContext, verbose=False) - if response.status == 200: - for line in response: - string_message = line.decode("utf-8") - print(string_message[string_message.index('=') + 1:]) + esautils.execute_servlet_request( + url=conf.JWST_TAP_SERVER + "/" + conf.JWST_MESSAGES, + tap=self.tap, + query_params={}, + parser_method=self.parse_messages_response + ) except OSError: print("Status messages could not be retrieved") + def parse_messages_response(self, response): + string_messages = [] + for line in response.iter_lines(): + string_message = line.decode("utf-8") + string_messages.append( + string_message[string_message.index('=') + 1:]) + print(string_messages[len(string_messages) - 1]) + return string_messages + def get_product_list(self, *, observation_id=None, cal_level="ALL", product_type=None): @@ -735,7 +652,8 @@ def get_product_list(self, *, observation_id=None, if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) - plane_ids, max_cal_level = self._get_plane_id(observation_id=observation_id) + plane_ids, max_cal_level = ( + self._get_plane_id(observation_id=observation_id)) if cal_level == 3 and cal_level > max_cal_level: raise ValueError("Requesting upper levels is not allowed") list = self._get_associated_planes(plane_ids=plane_ids, @@ -743,14 +661,16 @@ def get_product_list(self, *, observation_id=None, max_cal_level=max_cal_level, is_url=False) + condition = self.__get_artifact_producttype_condition( + product_type=product_type + ) + query = (f"select distinct a.uri, a.artifactid, a.filename, " f"a.contenttype, a.producttype, p.calibrationlevel, " f"p.public FROM {conf.JWST_PLANE_TABLE} p JOIN " f"{conf.JWST_ARTIFACT_TABLE} a ON (p.planeid=a.planeid) " - f"WHERE a.planeid IN {list}" - f"{self.__get_artifact_producttype_condition(product_type=product_type)};") - job = self.__jwsttap.launch_job(query=query) - return job.get_results() + f"WHERE a.planeid IN {list} {condition};") + return self.query_tap(query=query) def __validate_cal_level(self, cal_level): if (cal_level not in self.CAL_LEVELS): @@ -767,8 +687,10 @@ def _get_associated_planes(self, plane_ids, cal_level, else: plane_list = [] for plane_id in plane_ids: - siblings = self.__get_sibling_planes(planeid=plane_id, cal_level=cal_level) - members = self.__get_member_planes(planeid=plane_id, cal_level=cal_level) + siblings = self.__get_sibling_planes(planeid=plane_id, + cal_level=cal_level) + members = self.__get_member_planes(planeid=plane_id, + cal_level=cal_level) plane_id_table = vstack([siblings, members]) plane_list.extend(plane_id_table['product_planeid']) if (not is_url): @@ -781,13 +703,13 @@ def _get_plane_id(self, observation_id): try: planeids = [] query_plane = (f"select distinct m.planeid, m.calibrationlevel " - f"from {conf.JWST_MAIN_TABLE} m where " + f"from {conf.JWST_ARCHIVE_TABLE} m where " f"m.observationid = '{observation_id}'") - job = self.__jwsttap.launch_job(query=query_plane) - job.get_results().sort(["calibrationlevel"]) - job.get_results().reverse() - max_cal_level = job.get_results()["calibrationlevel"][0] - for row in job.get_results(): + results = self.query_tap(query=query_plane) + results.sort(["calibrationlevel"]) + results.reverse() + max_cal_level = results["calibrationlevel"][0] + for row in results: if row["calibrationlevel"] == max_cal_level: planeids.append( JwstClass.get_decoded_string(row["planeid"])) @@ -814,8 +736,7 @@ def __get_sibling_planes(self, planeid, *, cal_level='ALL'): f"p.obsid=o.obsid JOIN " f"{conf.JWST_PLANE_TABLE} sp ON " f"sp.obsid=o.obsid {where_clause}'{planeid}'") - job = self.__jwsttap.launch_job(query=query_siblings) - return job.get_results() + return self.query_tap(query=query_siblings) except Exception as e: raise ValueError(e) @@ -841,8 +762,7 @@ def __get_member_planes(self, planeid, *, cal_level='ALL'): f"{conf.JWST_PLANE_TABLE} mp on " f"mo.obsid=mp.obsid " f"{where_clause}'{planeid}'") - job = self.__jwsttap.launch_job(query=query_members) - return job.get_results() + return self.query_tap(query=query_members) except Exception as e: raise ValueError(e) @@ -865,19 +785,19 @@ def get_related_observations(self, observation_id): """ if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) - query_upper = (f"select * from {conf.JWST_MAIN_TABLE} m " + query_upper = (f"select * from {conf.JWST_ARCHIVE_TABLE} m " f"where m.members like " f"'%{observation_id}%'") - job = self.__jwsttap.launch_job(query=query_upper) - if any(job.get_results()["observationid"]): - oids = job.get_results()["observationid"] + results = self.query_tap(query=query_upper) + if any(results["observationid"]): + oids = results["observationid"] else: - query_members = (f"select m.members from {conf.JWST_MAIN_TABLE} " + query_members = (f"select m.members from {conf.JWST_ARCHIVE_TABLE} " f"m where m.observationid" f"='{observation_id}'") - job = self.__jwsttap.launch_job(query=query_members) + results = self.query_tap(query=query_members) oids = JwstClass.get_decoded_string( - job.get_results()["members"][0]).\ + results["members"][0]).\ replace("caom:JWST/", "").split(" ") return oids @@ -905,10 +825,12 @@ def get_product(self, *, artifact_id=None, file_name=None): if file_name is None: try: - output_file_name = self._query_get_product(artifact_id=artifact_id) + output_file_name = self._query_get_product( + artifact_id=artifact_id) err_msg = str(artifact_id) except Exception as exx: - raise ValueError(f"Cannot retrieve product for artifact_id {artifact_id}: {exx}") + raise ValueError(f"Cannot retrieve product " + f"for artifact_id {artifact_id}: {exx}") else: output_file_name = str(file_name) err_msg = str(file_name) @@ -920,11 +842,14 @@ def get_product(self, *, artifact_id=None, file_name=None): params_dict['ARTIFACTID'] = (self._query_get_product( file_name=file_name)) except Exception as exx: - raise ValueError(f"Cannot retrieve product for file_name {file_name}: {exx}") + raise ValueError(f"Cannot retrieve product " + f"for file_name {file_name}: {exx}") try: - self.__jwsttap.load_data(params_dict=params_dict, - output_file=output_file_name) + esautils.download_file(url=conf.JWST_DATA_SERVER, + session=self.tap._session, + params=params_dict, + filename=output_file_name) except Exception as exx: log.info("error") raise ValueError(f"Error retrieving product for {err_msg}: {exx}") @@ -936,16 +861,16 @@ def _query_get_product(self, *, artifact_id=None, file_name=None): query_artifactid = (f"select * from {conf.JWST_ARTIFACT_TABLE} " f"a where a.filename = " f"'{file_name}'") - job = self.__jwsttap.launch_job(query=query_artifactid) + results = self.query_tap(query=query_artifactid) return JwstClass.get_decoded_string( - job.get_results()['artifactid'][0]) + results['artifactid']) else: query_filename = (f"select * from {conf.JWST_ARTIFACT_TABLE} a " f"where a.artifactid = " f"'{artifact_id}'") - job = self.__jwsttap.launch_job(query=query_filename) + results = self.query_tap(query=query_filename) return JwstClass.get_decoded_string( - job.get_results()['filename'][0]) + results['filename']) def __check_product_input(self, artifact_id, file_name): if artifact_id is None and file_name is None: @@ -969,10 +894,12 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", levels, please use get_related_observations functions first. Possible values: 'ALL', 3, 2, 1, -1 product_type : str or list, optional, default None - If the string or at least one element of the list is empty, the value is replaced by None. + If the string or at least one element of the list is empty, + the value is replaced by None. With None, all products will be downloaded. - Possible string values: 'thumbnail', 'preview', 'auxiliary', 'science' or 'info'. - Posible list values: any combination of string values. + Possible string values: 'thumbnail', 'preview', + 'auxiliary', 'science' or 'info'. + Possible list values: any combination of string values. output_file : str, optional Output file. If no value is provided, a temporary one is created. @@ -982,11 +909,14 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", Returns the local path where the product(s) are saved. """ - if (isinstance(product_type, list) and '' in product_type) or not product_type: + if ( + isinstance(product_type, list) and '' in product_type + ) or not product_type: product_type = None if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) - plane_ids, max_cal_level = self._get_plane_id(observation_id=observation_id) + plane_ids, max_cal_level = self._get_plane_id( + observation_id=observation_id) if (cal_level == 3 and cal_level > max_cal_level): raise ValueError("Requesting upper levels is not allowed") @@ -1010,16 +940,20 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", cal_level=cal_level, max_cal_level=max_cal_level, product_type=tap_product_type) - output_file_full_path, output_dir = self.__set_dirs(output_file=output_file, - observation_id=observation_id) + (output_file_full_path, + output_dir) = self.__set_dirs(output_file=output_file, + observation_id=observation_id) # Get file name only output_file_name = os.path.basename(output_file_full_path) try: - self.__jwsttap.load_data(params_dict=params_dict, - output_file=output_file_full_path) + esautils.download_file(url=conf.JWST_DATA_SERVER, + session=self.tap._session, + params=params_dict, + filename=output_file_full_path) except Exception as exx: - raise ValueError(f"Cannot retrieve products for observation {observation_id}: {exx}") + raise ValueError(f"Cannot retrieve products " + f"for observation {observation_id}: {exx}") files = [] self.__extract_file(output_file_full_path=output_file_full_path, @@ -1035,7 +969,11 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", return files - def download_files_from_program(self, proposal_id, *, product_type=None, verbose=False): + def download_files_from_program(self, + proposal_id, + *, + product_type=None, + verbose=False): """Get JWST products given its proposal ID. Parameters @@ -1046,7 +984,8 @@ def download_files_from_program(self, proposal_id, *, product_type=None, verbose If the string or at least one element of the list is empty, the value is replaced by None. With None, all products will be downloaded. - Possible string values: 'thumbnail', 'preview', 'auxiliary', 'science' or 'info'. + Possible string values: 'thumbnail', 'preview', + 'auxiliary', 'science' or 'info'. Posible list values: any combination of string values. verbose : bool, optional, default 'False' flag to display information about the process @@ -1062,11 +1001,13 @@ def download_files_from_program(self, proposal_id, *, product_type=None, verbose f"WHERE proposal_id='{str(proposal_id)}'") if verbose: print(query) - job = self.__jwsttap.launch_job_async(query=query, verbose=verbose) - allobs = set(JwstClass.get_decoded_string(job.get_results()['observationid'])) + results = self.query_tap(query=query, verbose=verbose) + allobs = set(JwstClass.get_decoded_string( + results['observationid'])) for oid in allobs: log.info(f"Downloading products for Observation ID: {oid}") - self.get_obs_products(observation_id=oid, product_type=product_type) + self.get_obs_products(observation_id=oid, + product_type=product_type) return list(allobs) def __check_file_number(self, output_dir, output_file_name, @@ -1116,7 +1057,8 @@ def __set_dirs(self, output_file, observation_id): try: os.makedirs(output_dir, exist_ok=True) except OSError as err: - raise OSError(f"Creation of the directory {output_dir} failed: {err.strerror}") + raise OSError(f"Creation of the directory {output_dir} " + f"failed: {err.strerror}") return output_file_full_path, output_dir def __set_additional_parameters(self, param_dict, cal_level, @@ -1137,7 +1079,8 @@ def __get_quantity_input(self, value, msg): if value is None: raise ValueError(f"Missing required argument: '{msg}'") if not (isinstance(value, str) or isinstance(value, units.Quantity)): - raise ValueError(f"{msg} must be either a string or units.Quantity") + raise ValueError(f"{msg} must be either a string " + f"or units.Quantity") if isinstance(value, str): q = Quantity(value) return q @@ -1147,7 +1090,8 @@ def __get_quantity_input(self, value, msg): def __get_coord_input(self, value, msg): if not (isinstance(value, str) or isinstance(value, commons.CoordClasses)): - raise ValueError(f"{msg} must be either a string or astropy.coordinates") + raise ValueError(f"{msg} must be either a string " + f"or astropy.coordinates") if isinstance(value, str): c = commons.parse_coordinates(value) return c @@ -1166,13 +1110,10 @@ def __get_observationid_condition(self, *, value=None): def __get_callevel_condition(self, cal_level): condition = "" if (cal_level is not None): - if (isinstance(cal_level, str) and cal_level == 'Top'): - condition = " AND max_cal_level=calibrationlevel " - elif (isinstance(cal_level, int)): + if isinstance(cal_level, int): condition = f" AND calibrationlevel={str(cal_level)} " else: - raise ValueError("cal_level must be either " - "'Top' or an integer") + raise ValueError("cal_level must be an integer") return condition def __get_public_condition(self, only_public): @@ -1189,9 +1130,12 @@ def __get_plane_dataproducttype_condition(self, *, prod_type=None): if not isinstance(prod_type, str): raise ValueError("prod_type must be string") elif str(prod_type).lower() not in self.PLANE_DATAPRODUCT_TYPES: - raise ValueError("prod_type must be one of: {str(', '.join(self.PLANE_DATAPRODUCT_TYPES))}") + raise ValueError("prod_type must be one of: " + "{str(', '.join(self." + "PLANE_DATAPRODUCT_TYPES))}") else: - condition = f" AND dataproducttype ILIKE '%{prod_type.lower()}%' " + condition = (f" AND dataproducttype " + f"ILIKE '%{prod_type.lower()}%' ") return condition def __get_instrument_name_condition(self, *, value=None): @@ -1200,7 +1144,11 @@ def __get_instrument_name_condition(self, *, value=None): if (not isinstance(value, str)): raise ValueError("instrument_name must be string") elif (str(value).upper() not in self.INSTRUMENT_NAMES): - raise ValueError(f"instrument_name must be one of: {str(', '.join(self.INSTRUMENT_NAMES))}") + msg = ( + "instrument_name must be one of: " + f"{', '.join(self.INSTRUMENT_NAMES)}" + ) + raise ValueError(msg) else: condition = f" AND instrument_name ILIKE '%{value.upper()}%' " return condition @@ -1231,11 +1179,29 @@ def __get_artifact_producttype_condition(self, *, product_type=None): if (not isinstance(product_type, str)): raise ValueError("product_type must be string") elif (product_type not in self.ARTIFACT_PRODUCT_TYPES): - raise ValueError(f"product_type must be one of: {str(', '.join(self.ARTIFACT_PRODUCT_TYPES))}") + msg = ( + "product_type must be one of: " + f"{', '.join(self.ARTIFACT_PRODUCT_TYPES)}" + ) + raise ValueError(msg) else: condition = f" AND producttype ILIKE '%{product_type}%'" return condition + def _fix_jwst_boolean_fields(self, votable): + """ + Convert bit to boolean for specific JWST columns, since + Upload servlet does not ingest Bit types + """ + + BOOLEAN_FIX_COLUMNS = {"public", "vis_cube", "vis_image"} + + table = votable.resources[0].tables[0] + for field in table.fields: + if field.name in BOOLEAN_FIX_COLUMNS: + if field.datatype == "bit": + field.datatype = "boolean" + @staticmethod def is_gz_file(filepath): with open(filepath, 'rb') as test_f: @@ -1275,4 +1241,6 @@ def get_decoded_string(str): return str -Jwst = JwstClass() +# Need to be False in order to avoid reaching out +# to the remote server at import time +Jwst = JwstClass(show_messages=False) diff --git a/astroquery/esa/jwst/data_access.py b/astroquery/esa/jwst/data_access.py deleted file mode 100644 index f39bd35d01..0000000000 --- a/astroquery/esa/jwst/data_access.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -""" -================= -eJWST Data Access -================= - -European Space Astronomy Centre (ESAC) -European Space Agency (ESA) - -""" - -from astropy.utils import data -from . import conf - -__all__ = ['JwstDataHandler'] - - -class JwstDataHandler: - def __init__(self, base_url=None): - if base_url is None: - self.base_url = conf.JWST_DATA_SERVER - else: - self.base_url = base_url - - def download_file(self, url): - return data.download_file(url, cache=True) - - def clear_download_cache(self): - data.clear_download_cache() diff --git a/astroquery/esa/jwst/tests/DummyTapHandler.py b/astroquery/esa/jwst/tests/DummyTapHandler.py index 8ed60e47f2..b6aed64632 100644 --- a/astroquery/esa/jwst/tests/DummyTapHandler.py +++ b/astroquery/esa/jwst/tests/DummyTapHandler.py @@ -48,7 +48,9 @@ def check_method(self, method): if method == self.__invokedMethod: return else: - raise ValueError(f"Method '+{str(method)}' not invoked. (Invoked method is '{str(self.__invokedMethod)}')") + raise ValueError(f"Method '+{str(method)}' " + f"not invoked. (Invoked method " + f"is '{str(self.__invokedMethod)}')") def check_parameters(self, parameters, method_name): print("FOUND") @@ -236,6 +238,11 @@ def logout(self, verbose=False): self.__parameters['verbose'] = verbose return None + def upload_table(self, *, verbose=False): + self.__invokedMethod = 'upload_table' + self.__parameters['verbose'] = verbose + return None + def load_data(self, params_dict, output_file=None, verbose=False): self.__invokedMethod = 'load_data' self.__parameters['params_dict'] = params_dict diff --git a/astroquery/esa/jwst/tests/test_jwstdata.py b/astroquery/esa/jwst/tests/test_jwstdata.py index 834b80e786..956121f31c 100644 --- a/astroquery/esa/jwst/tests/test_jwstdata.py +++ b/astroquery/esa/jwst/tests/test_jwstdata.py @@ -9,10 +9,12 @@ """ import os +from unittest.mock import patch + import pytest from requests import HTTPError -from astroquery.esa.jwst.tests.DummyTapHandler import DummyTapHandler +from astroquery.esa.jwst import conf from astroquery.esa.jwst.core import JwstClass @@ -37,32 +39,42 @@ def get_product_request(request): class TestData: - def test_get_product(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - # default parameters - parameters = {} - parameters['artifact_id'] = None + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.jwst.core.esautils.download_file') + @patch.object(JwstClass, '_query_get_product', + return_value='jw00617023001_02102_00001_nrcb4_uncal.fits') + def test_get_product(self, mock_qget, mock_download): + jwst = JwstClass(show_messages=False) + with pytest.raises(ValueError) as err: jwst.get_product() - assert "Missing required argument: 'artifact_id'" in err.value.args[0] - # test with parameters - dummyTapHandler.reset() - parameters = {} - params_dict = {} - params_dict['RETRIEVAL_TYPE'] = 'PRODUCT' - params_dict['TAPCLIENT'] = 'ASTROQUERY' - params_dict['ARTIFACTID'] = '00000000-0000-0000-8740-65e2827c9895' - parameters['params_dict'] = params_dict - parameters['output_file'] = 'jw00617023001_02102_00001_nrcb4_uncal.fits' - parameters['verbose'] = False - jwst.get_product(artifact_id='00000000-0000-0000-8740-65e2827c9895') - dummyTapHandler.check_call('load_data', parameters) + assert "Missing required argument: 'artifact_id'" in str(err.value) + + artifact_id = "00000000-0000-0000-8740-65e2827c9895" + mock_download.return_value = \ + "/tmp/jw00617023001_02102_00001_nrcb4_uncal.fits" + + result = jwst.get_product(artifact_id=artifact_id) + + mock_qget.assert_called_once_with(artifact_id=artifact_id) + mock_download.assert_called_once() + _, kwargs = mock_download.call_args + + assert kwargs["url"] == conf.JWST_DATA_SERVER + assert kwargs["session"] is jwst.tap._session + assert kwargs["params"] == { + "RETRIEVAL_TYPE": "PRODUCT", + "TAPCLIENT": "ASTROQUERY", + "ARTIFACTID": artifact_id, + } + expected_filename = "jw00617023001_02102_00001_nrcb4_uncal.fits" + assert kwargs["filename"] == expected_filename + assert result is not None @pytest.mark.remote_data def test_login_error(): - jwst = JwstClass() + jwst = JwstClass(show_messages=False) with pytest.raises(HTTPError) as err: jwst.login(user="dummy", password="dummy") - assert "Unauthorized" in err.value.args[0] + assert "401" in err.value.args[0] diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index d627c7b8e2..4e3688839e 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -11,10 +11,12 @@ import os import shutil from pathlib import Path -from unittest.mock import MagicMock, patch +from types import SimpleNamespace +from unittest.mock import MagicMock, patch, call, Mock import sys import io + import astropy.units as u import numpy as np import pytest @@ -24,15 +26,13 @@ from astropy.io.votable import parse_single_table from astropy.table import Table from astropy.units import Quantity +from requests import Response + from astroquery.exceptions import TableParseError from astroquery.esa.jwst import JwstClass -from astroquery.esa.jwst.tests.DummyTapHandler import DummyTapHandler from astroquery.ipac.ned import Ned from astroquery.simbad import SimbadClass -from astroquery.utils.tap.conn.tests.DummyConnHandler import DummyConnHandler -from astroquery.utils.tap.conn.tests.DummyResponse import DummyResponse -from astroquery.utils.tap.core import TapPlus from astroquery.vizier import Vizier from astroquery.esa.jwst import conf @@ -87,172 +87,94 @@ def get_product_request(request): return mp -planeids = "('00000000-0000-0000-879d-ae91fa2f43e2', '00000000-0000-0000-9852-a9fa8c63f7ef')" +planeids = ("(" + "'00000000-0000-0000-879d-ae91fa2f43e2', " + "'00000000-0000-0000-9852-a9fa8c63f7ef'" + ")") class TestTap: - def test_load_tables(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - # default parameters - parameters = {} - parameters['only_names'] = False - parameters['include_shared_tables'] = False - parameters['verbose'] = False - tap.load_tables() - dummyTapHandler.check_call('load_tables', parameters) - # test with parameters - dummyTapHandler.reset() - parameters = {} - parameters['only_names'] = True - parameters['include_shared_tables'] = True - parameters['verbose'] = True - tap.load_tables(only_names=True, include_shared_tables=True, verbose=True) - dummyTapHandler.check_call('load_tables', parameters) - - def test_load_table(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - # default parameters - parameters = {} - parameters['table'] = 'table' - parameters['verbose'] = False - tap.load_table('table') - dummyTapHandler.check_call('load_table', parameters) - # test with parameters - dummyTapHandler.reset() - parameters = {} - parameters['table'] = 'table' - parameters['verbose'] = True - tap.load_table('table', verbose=True) - dummyTapHandler.check_call('load_table', parameters) - - def test_launch_sync_job(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + def test_launch_sync_job(self, mock_query_tap): + tap = JwstClass(show_messages=False) query = "query" # default parameters - parameters = {} - parameters['query'] = query - parameters['name'] = None - parameters['output_file'] = None - parameters['output_format'] = 'votable' - parameters['verbose'] = False - parameters['dump_to_file'] = False - parameters['upload_resource'] = None - parameters['upload_table_name'] = None tap.launch_job(query) - dummyTapHandler.check_call('launch_job', parameters) + # test with parameters - dummyTapHandler.reset() - name = 'name' - output_file = 'output' - output_format = 'format' - verbose = True - dump_to_file = True - upload_resource = 'upload_res' - upload_table_name = 'upload_table' + parameters = {} parameters['query'] = query - parameters['name'] = name - parameters['output_file'] = output_file - parameters['output_format'] = output_format - parameters['verbose'] = verbose - parameters['dump_to_file'] = dump_to_file - parameters['upload_resource'] = upload_resource - parameters['upload_table_name'] = upload_table_name - tap.launch_job(query, - name=name, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file, - upload_resource=upload_resource, - upload_table_name=upload_table_name) - dummyTapHandler.check_call('launch_job', parameters) - - def test_launch_async_job(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + parameters['async_job'] = False + parameters['output_file'] = 'output' + parameters['output_format'] = 'format' + parameters['verbose'] = True + tap.launch_job(**parameters) + + assert mock_query_tap.call_args_list == [ + call('query', + async_job=False, + output_file=None, + output_format='votable', + verbose=False), + call('query', + async_job=False, + output_file='output', + output_format='format', + verbose=True), + ] + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + def test_launch_async_job(self, mock_query_tap): + tap = JwstClass(show_messages=False) query = "query" # default parameters parameters = {} parameters['query'] = query - parameters['name'] = None + parameters['async_job'] = True parameters['output_file'] = None parameters['output_format'] = 'votable' parameters['verbose'] = False - parameters['dump_to_file'] = False - parameters['background'] = False - parameters['upload_resource'] = None - parameters['upload_table_name'] = None - tap.launch_job(query, async_job=True) - dummyTapHandler.check_call('launch_job_async', parameters) - # test with parameters - dummyTapHandler.reset() - name = 'name' - output_file = 'output' - output_format = 'format' - verbose = True - dump_to_file = True - background = True - upload_resource = 'upload_res' - upload_table_name = 'upload_table' - parameters['query'] = query - parameters['name'] = name - parameters['output_file'] = output_file - parameters['output_format'] = output_format - parameters['verbose'] = verbose - parameters['dump_to_file'] = dump_to_file - parameters['background'] = background - parameters['upload_resource'] = upload_resource - parameters['upload_table_name'] = upload_table_name - tap.launch_job(query, - name=name, - output_file=output_file, - output_format=output_format, - verbose=verbose, - dump_to_file=dump_to_file, - background=background, - upload_resource=upload_resource, - upload_table_name=upload_table_name, - async_job=True) - dummyTapHandler.check_call('launch_job_async', parameters) - - def test_list_async_jobs(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - # default parameters - parameters = {} - parameters['verbose'] = False - tap.list_async_jobs() - dummyTapHandler.check_call('list_async_jobs', parameters) - # test with parameters - dummyTapHandler.reset() - parameters['verbose'] = True - tap.list_async_jobs(verbose=True) - dummyTapHandler.check_call('list_async_jobs', parameters) + tap.launch_job(**parameters) - def test_query_region(self): - connHandler = DummyConnHandler() - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) + # test with set parameters + parameters['output_file'] = 'output' + parameters['output_format'] = 'format' + parameters['verbose'] = True + tap.launch_job(**parameters) + + assert mock_query_tap.call_args_list == [ + call('query', + async_job=True, + output_file=None, + output_format='votable', + verbose=False), + call('query', + async_job=True, + output_file='output', + output_format='format', + verbose=True), + ] + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch("astroquery.esa.utils.utils.pyvo.dal.TAPService.search") + def test_query_region(self, mock_query_tap): + tap = JwstClass(show_messages=False) with pytest.raises(ValueError) as err: tap.query_region(coordinate=123) - assert "coordinate must be either a string or astropy.coordinates" in err.value.args[0] + assert ("coordinate must be either a string " + "or astropy.coordinates") in err.value.args[0] with pytest.raises(NameResolveError) as err: tap.query_region(coordinate='test') - assert ("Unable to find coordinates for name 'test'" in err.value.args[0] or "Unable to retrieve " + assert ("Unable to find coordinates for name " + "'test'" in err.value.args[0] or "Unable to retrieve " "coordinates" in err.value.args[0]) - # Launch response: we use default response because the - # query contains decimals - responseLaunchJob = DummyResponse(200) - responseLaunchJob.set_data(method='POST', body=JOB_DATA) # The query contains decimals: force default response - connHandler.set_default_response(responseLaunchJob) sc = SkyCoord(ra=29.0, dec=15.0, unit=(u.degree, u.degree), frame='icrs') with pytest.raises(ValueError) as err: @@ -262,7 +184,8 @@ def test_query_region(self): width = 123 with pytest.raises(ValueError) as err: tap.query_region(sc, width=width) - assert "width must be either a string or units.Quantity" in err.value.args[0] + assert ("width must be either a string " + "or units.Quantity") in err.value.args[0] width = Quantity(12, u.deg) height = Quantity(10, u.deg) @@ -271,30 +194,50 @@ def test_query_region(self): tap.query_region(sc, width=width) assert "Missing required argument: 'height'" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height), Table)) + votable = parse_single_table(io.BytesIO(JOB_DATA.encode("utf-8"))) + results_table = votable.to_table(use_names_over_ids=True) + + # Mock job object returned by query_tap() + mock_result = Mock() + mock_result.to_table.return_value = results_table + mock_query_tap.return_value = mock_result + + assert (isinstance(tap.query_region(sc, + width=width, + height=height), Table)) # Test observation_id argument with pytest.raises(ValueError) as err: tap.query_region(sc, width=width, height=height, observation_id=1) assert "observation_id must be string" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, observation_id="observation"), Table)) - # raise ValueError + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + observation_id="observation"), + Table)) # Test cal_level argument with pytest.raises(ValueError) as err: - tap.query_region(sc, width=width, height=height, cal_level='a') - assert "cal_level must be either 'Top' or an integer" in err.value.args[0] - - assert (isinstance(tap.query_region(sc, width=width, height=height, cal_level='Top'), Table)) - assert (isinstance(tap.query_region(sc, width=width, height=height, cal_level=1), Table)) + tap.query_region(sc, + width=width, + height=height, + cal_level='a') + assert ("cal_level must be an integer") in err.value.args[0] + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + cal_level=1), Table)) # Test only_public with pytest.raises(ValueError) as err: tap.query_region(sc, width=width, height=height, only_public='a') assert "only_public must be boolean" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, only_public=True), Table)) + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + only_public=True), Table)) # Test dataproduct_type argument with pytest.raises(ValueError) as err: @@ -302,17 +245,26 @@ def test_query_region(self): assert "prod_type must be string" in err.value.args[0] with pytest.raises(ValueError) as err: - tap.query_region(sc, width=width, height=height, prod_type='a') + tap.query_region(sc, + width=width, + height=height, + prod_type='a') assert "prod_type must be one of: " in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, prod_type='image'), Table)) + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + prod_type='image'), Table)) # Test instrument_name argument with pytest.raises(ValueError) as err: tap.query_region(sc, width=width, height=height, instrument_name=1) assert "instrument_name must be string" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, instrument_name='NIRCAM'), Table)) + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + instrument_name='NIRCAM'), Table)) with pytest.raises(ValueError) as err: tap.query_region(sc, width=width, height=height, @@ -324,17 +276,24 @@ def test_query_region(self): tap.query_region(sc, width=width, height=height, filter_name=1) assert "filter_name must be string" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, filter_name='filter'), Table)) + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + filter_name='filter'), Table)) # Test proposal_id argument with pytest.raises(ValueError) as err: tap.query_region(sc, width=width, height=height, proposal_id=123) assert "proposal_id must be string" in err.value.args[0] - assert (isinstance(tap.query_region(sc, width=width, height=height, proposal_id='123'), Table)) + assert (isinstance(tap.query_region(sc, + width=width, + height=height, + proposal_id='123'), Table)) table = tap.query_region(sc, width=width, height=height) - assert len(table) == 3, f"Wrong job results (num rows). Expected: {3}, found {len(table)}" + assert len(table) == 3, (f"Wrong job results (num rows). " + f"Expected: {3}, found {len(table)}") self.__check_results_column(table, 'alpha', 'alpha', @@ -358,7 +317,8 @@ def test_query_region(self): # by radius radius = Quantity(1, u.deg) table = tap.query_region(sc, radius=radius) - assert len(table) == 3, f"Wrong job results (num rows). Expected: {3}, found {len(table)}" + assert len(table) == 3, (f"Wrong job results (num rows). " + f"Expected: {3}, found {len(table)}") self.__check_results_column(table, 'alpha', 'alpha', @@ -380,33 +340,29 @@ def test_query_region(self): None, np.int32) - def test_query_region_async(self): - connHandler = DummyConnHandler() - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) - jobid = '12345' - # Launch response - responseLaunchJob = DummyResponse(303) - # list of list (httplib implementation for headers in response) - launchResponseHeaders = [['location', 'http://test:1111/tap/async/' + jobid]] - responseLaunchJob.set_data(method='POST', headers=launchResponseHeaders) - connHandler.set_default_response(responseLaunchJob) - # Phase response - responsePhase = DummyResponse(200) - responsePhase.set_data(method='GET', body="COMPLETED") - req = "async/" + jobid + "/phase" - connHandler.set_response(req, responsePhase) - # Results response - responseResultsJob = DummyResponse(200) - responseResultsJob.set_data(method='GET', body=JOB_DATA) - req = "async/" + jobid + "/results/result" - connHandler.set_response(req, responseResultsJob) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch("astroquery.esa.utils.utils.pyvo.dal.TAPService.run_async") + def test_query_region_async(self, mock_query_tap): + tap = JwstClass(show_messages=False) + + votable = parse_single_table(io.BytesIO(JOB_DATA.encode('utf-8'))) + results_table = votable.to_table(use_names_over_ids=True) + + # Mock job object returned by query_tap() + mock_job = Mock() + mock_job.to_table.return_value = results_table + mock_query_tap.return_value = mock_job + sc = SkyCoord(ra=29.0, dec=15.0, unit=(u.degree, u.degree), frame='icrs') width = Quantity(12, u.deg) height = Quantity(10, u.deg) - table = tap.query_region(sc, width=width, height=height, async_job=True) - assert len(table) == 3, f"Wrong job results (num rows). Expected: {3}, found {len(table)}" + table = tap.query_region(sc, + width=width, + height=height, + async_job=True) + assert len(table) == 3, (f"Wrong job results (num rows). " + f"Expected: {3}, found {len(table)}") self.__check_results_column(table, 'alpha', 'alpha', @@ -427,10 +383,14 @@ def test_query_region_async(self): 'table1_oid', None, np.int32) + mock_query_tap.assert_called_once() + + mock_query_tap.reset_mock() # by radius radius = Quantity(1, u.deg) table = tap.query_region(sc, radius=radius, async_job=True) - assert len(table) == 3, f"Wrong job results (num rows). Expected: {3}, found {len(table)}" + assert len(table) == 3, (f"Wrong job results (num rows). " + f"Expected: {3}, found {len(table)}") self.__check_results_column(table, 'alpha', 'alpha', @@ -451,28 +411,40 @@ def test_query_region_async(self): 'table1_oid', None, np.int32) + mock_query_tap.assert_called_once() + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + def test_cone_search_sync(self, mock_query_tap): + tap = JwstClass(show_messages=False) - def test_cone_search_sync(self): - connHandler = DummyConnHandler() - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) - # Launch response: we use default response because the - # query contains decimals - responseLaunchJob = DummyResponse(200) - responseLaunchJob.set_data(method='POST', body=JOB_DATA) + mock_job = MagicMock() + mock_job.async_ = False + mock_job.failed = False + mock_job.get_phase.return_value = 'COMPLETED' + + votable = parse_single_table(io.BytesIO(JOB_DATA.encode("utf-8"))) + results_table = votable.to_table(use_names_over_ids=True) + + mock_job.get_results.return_value = results_table + + # query_tap returns this job + mock_query_tap.return_value = mock_job ra = 19.0 dec = 20.0 sc = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs') radius = Quantity(1.0, u.deg) - connHandler.set_default_response(responseLaunchJob) job = tap.cone_search(sc, radius) assert job is not None, "Expected a valid job" assert job.async_ is False, "Expected a synchronous job" - assert job.get_phase() == 'COMPLETED', f"Wrong job phase. Expected: {'COMPLETED'}, found {job.get_phase()}" + assert job.get_phase() == 'COMPLETED', (f"Wrong job phase. " + f"Expected: {'COMPLETED'}, " + f"found {job.get_phase()}") assert job.failed is False, "Wrong job status (set Failed = True)" # results results = job.get_results() - assert len(results) == 3, f"Wrong job results (num rows). Expected: {3}, found {len(results)}" + assert len(results) == 3, (f"Wrong job results (num rows). " + f"Expected: {3}, found {len(results)}") self.__check_results_column(results, 'alpha', 'alpha', @@ -502,7 +474,7 @@ def test_cone_search_sync(self): # Test cal_level argument with pytest.raises(ValueError) as err: tap.cone_search(sc, radius, cal_level='a') - assert "cal_level must be either 'Top' or an integer" in err.value.args[0] + assert ("cal_level must be an integer") in err.value.args[0] # Test only_public with pytest.raises(ValueError) as err: @@ -537,39 +509,42 @@ def test_cone_search_sync(self): tap.cone_search(sc, radius, proposal_id=123) assert "proposal_id must be string" in err.value.args[0] - def test_cone_search_async(self): - connHandler = DummyConnHandler() - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) - jobid = '12345' - # Launch response - responseLaunchJob = DummyResponse(303) - # list of list (httplib implementation for headers in response) - launchResponseHeaders = [['location', 'http://test:1111/tap/async/' + jobid]] - responseLaunchJob.set_data(method='POST', headers=launchResponseHeaders) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + def test_cone_search_async(self, mock_query_tap): + tap = JwstClass(show_messages=False) + + mock_job = MagicMock() + mock_job.async_ = True + mock_job.failed = False + mock_job.get_phase.return_value = 'COMPLETED' + + votable = parse_single_table(io.BytesIO(JOB_DATA.encode("utf-8"))) + results_table = votable.to_table(use_names_over_ids=True) + + mock_job.get_results.return_value = results_table + + # query_tap returns this job + mock_query_tap.return_value = mock_job + ra = 19 dec = 20 sc = SkyCoord(ra=ra, dec=dec, unit=(u.degree, u.degree), frame='icrs') radius = Quantity(1.0, u.deg) - connHandler.set_default_response(responseLaunchJob) - # Phase response - responsePhase = DummyResponse(200) - responsePhase.set_data(method='GET', body="COMPLETED") - req = "async/" + jobid + "/phase" - connHandler.set_response(req, responsePhase) - # Results response - responseResultsJob = DummyResponse(200) - responseResultsJob.set_data(method='GET', body=JOB_DATA) - req = "async/" + jobid + "/results/result" - connHandler.set_response(req, responseResultsJob) + job = tap.cone_search(sc, radius, async_job=True) + + assert job is mock_job assert job is not None, "Expected a valid job" assert job.async_ is True, "Expected an asynchronous job" - assert job.get_phase() == 'COMPLETED', f"Wrong job phase. Expected: {'COMPLETED'}, found {job.get_phase()}" + assert job.get_phase() == 'COMPLETED', (f"Wrong job phase. " + f"Expected: {'COMPLETED'}, " + f"found {job.get_phase()}") assert job.failed is False, "Wrong job status (set Failed = True)" # results results = job.get_results() - assert len(results) == 3, "Wrong job results (num rows). Expected: {3}, found {len(results)}" + assert len(results) == 3, ("Wrong job results (num rows). " + "Expected: {3}, found {len(results)}") self.__check_results_column(results, 'alpha', 'alpha', @@ -591,19 +566,21 @@ def test_cone_search_async(self): None, np.int32) - def test_get_product_by_artifactid(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + def test_get_product_by_artifactid(self, download_mock): + jwst = JwstClass(show_messages=False) # default parameters + with pytest.raises(ValueError) as err: jwst.get_product() - assert "Missing required argument: 'artifact_id' or 'file_name'" in err.value.args[0] + assert ("Missing required argument: 'artifact_id' or " + "'file_name'") in err.value.args[0] # test with parameters - dummyTapHandler.reset() - parameters = {} - parameters['output_file'] = 'jw00617023001_02102_00001_nrcb4_uncal.fits' + parameters['output_file'] = \ + 'jw00617023001_02102_00001_nrcb4_uncal.fits' parameters['verbose'] = False param_dict = {} @@ -613,19 +590,33 @@ def test_get_product_by_artifactid(self): parameters['params_dict'] = param_dict jwst.get_product(artifact_id='00000000-0000-0000-8740-65e2827c9895') - dummyTapHandler.check_call('load_data', parameters) - - def test_get_product_by_filename(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + assert download_mock.call_count == 1 + + # Check the arguments passed to download_file + args, kwargs = download_mock.call_args + + expected_artifact_id = "00000000-0000-0000-8740-65e2827c9895" + assert kwargs["params"]["ARTIFACTID"] == expected_artifact_id + assert kwargs["params"]["RETRIEVAL_TYPE"] == "PRODUCT" + assert kwargs["params"]["TAPCLIENT"] == "ASTROQUERY" + + expected_filename = "jw00617023001_02102_00001_nrcb4_uncal.fits" + assert kwargs["filename"] == expected_filename + assert kwargs["url"].startswith( + "https://jwst.esac.esa.int/server/data") + assert "session" in kwargs + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + def test_get_product_by_filename(self, download_mock): + jwst = JwstClass(show_messages=False) # default parameters with pytest.raises(ValueError) as err: jwst.get_product() - assert "Missing required argument: 'artifact_id' or 'file_name'" in err.value.args[0] + assert ("Missing required argument: 'artifact_id' or " + "'file_name'") in err.value.args[0] # test with parameters - dummyTapHandler.reset() - parameters = {} parameters['output_file'] = 'file_name_id' parameters['verbose'] = False @@ -637,76 +628,90 @@ def test_get_product_by_filename(self): parameters['params_dict'] = param_dict jwst.get_product(file_name='file_name_id') - dummyTapHandler.check_call('load_data', parameters) + assert download_mock.call_count == 1 - def test_get_products_list(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + # Check the arguments passed to download_file + args, kwargs = download_mock.call_args - # test with parameters - dummyTapHandler.reset() + expected_id = "00000000-0000-0000-8740-65e2827c9895" + assert kwargs["params"]["ARTIFACTID"] == expected_id + assert kwargs["params"]["RETRIEVAL_TYPE"] == "PRODUCT" + assert kwargs["params"]["TAPCLIENT"] == "ASTROQUERY" + assert kwargs["filename"] == "file_name_id" + assert kwargs["url"].startswith( + "https://jwst.esac.esa.int/server/data") + assert "session" in kwargs + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch("astroquery.esa.utils.utils.pyvo.dal.TAPService.search") + def test_get_products_list(self, mock_query_tap): + jwst = JwstClass(show_messages=False) observation_id = "jw00777011001_02104_00001_nrcblong" - query = (f"select distinct a.uri, a.artifactid, a.filename, " - f"a.contenttype, a.producttype, p.calibrationlevel, p.public " - f"FROM {conf.JWST_PLANE_TABLE} p JOIN {conf.JWST_ARTIFACT_TABLE} " - f"a ON (p.planeid=a.planeid) WHERE a.planeid " - f"IN {planeids} AND producttype ILIKE '%science%';") + # Mock job and table resultsßß + mock_result = Mock() + expected_table = {"filename": ["f1.fits", "f2.fits"]} + mock_result.to_table.return_value = expected_table + mock_query_tap.return_value = mock_result - parameters = {} - parameters['query'] = query - parameters['name'] = None - parameters['output_file'] = None - parameters['output_format'] = 'votable' - parameters['verbose'] = False - parameters['dump_to_file'] = False - parameters['upload_resource'] = None - parameters['upload_table_name'] = None + result = jwst.get_product_list(observation_id=observation_id, + product_type="science") - jwst.get_product_list(observation_id=observation_id, product_type='science') - dummyTapHandler.check_call('launch_job', parameters) + mock_query_tap.assert_called_once() + assert result == expected_table + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) def test_get_products_list_error(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + jwst = JwstClass(show_messages=False) observation_id = "jw00777011001_02104_00001_nrcblong" + # default parameters with pytest.raises(ValueError) as err: jwst.get_product_list() - assert "Missing required argument: 'observation_id'" in err.value.args[0] + assert ("Missing required argument: " + "'observation_id'") in err.value.args[0] with pytest.raises(ValueError) as err: - jwst.get_product_list(observation_id=observation_id, product_type=1) + jwst.get_product_list(observation_id=observation_id, + product_type=1) assert "product_type must be string" in err.value.args[0] with pytest.raises(ValueError) as err: - jwst.get_product_list(observation_id=observation_id, product_type='test') + jwst.get_product_list(observation_id=observation_id, + product_type='test') assert "product_type must be one of" in err.value.args[0] + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) def test_download_files_from_program(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + jwst = JwstClass(show_messages=False) with pytest.raises(TypeError) as err: jwst.download_files_from_program() - assert "missing 1 required positional argument: 'proposal_id'" in err.value.args[0] - - def test_get_obs_products(self): - dummyTapHandler = DummyTapHandler() - jwst = JwstClass(tap_plus_handler=dummyTapHandler, data_handler=dummyTapHandler, show_messages=False) + assert ("missing 1 required positional argument: " + "'proposal_id'") in err.value.args[0] + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.download_file') + def test_get_obs_products(self, download_mock): + get_obs_products_counter = 0 + jwst = JwstClass(show_messages=False) # default parameters with pytest.raises(ValueError) as err: jwst.get_obs_products() - assert "Missing required argument: 'observation_id'" in err.value.args[0] + assert ("Missing required argument: " + "'observation_id'") in err.value.args[0] # test with parameters - dummyTapHandler.reset() - output_file_full_path_dir = os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_1" + output_file_full_path_dir = os.path.join( + os.getcwd(), + "temp_test_jwsttap_get_obs_products_1") try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err observation_id = 'jw00777011001_02104_00001_nrcblong' @@ -723,38 +728,50 @@ def test_get_obs_products(self): # Test single product tar file = data_path('single_product_retrieval.tar') - output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + output_file_full_path = os.path.join( + output_file_full_path_dir, + os.path.basename(file)) shutil.copy(file, output_file_full_path) parameters['output_file'] = output_file_full_path expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval_1.fits' + extracted_file_1 = os.path.join( + output_file_full_path_dir, + 'single_product_retrieval_1.fits') expected_files.append(extracted_file_1) try: files_returned = (jwst.get_obs_products( observation_id=observation_id, cal_level='ALL', output_file=output_file_full_path)) - dummyTapHandler.check_call('load_data', parameters) self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: shutil.rmtree(output_file_full_path_dir) # Test product_type paramater with a list - output_file_full_path_dir = os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_1" + output_file_full_path_dir = os.path.join( + os.getcwd(), + "temp_test_jwsttap_get_obs_products_1") try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('single_product_retrieval.tar') - output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + output_file_full_path = os.path.join( + output_file_full_path_dir, + os.path.basename(file)) shutil.copy(file, output_file_full_path) parameters['output_file'] = output_file_full_path expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval_1.fits' + extracted_file_1 = os.path.join( + output_file_full_path_dir, + 'single_product_retrieval_1.fits') expected_files.append(extracted_file_1) product_type_as_list = ['science', 'info'] try: @@ -764,9 +781,15 @@ def test_get_obs_products(self): product_type=product_type_as_list, output_file=output_file_full_path)) parameters['params_dict']['product_type'] = 'science,info' - dummyTapHandler.check_call('load_data', parameters) + + args, kwargs = download_mock.call_args + assert kwargs["params"]["product_type"] == "science,info" + assert kwargs["params"]["RETRIEVAL_TYPE"] == "OBSERVATION" + assert kwargs["filename"] == output_file_full_path + self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: shutil.rmtree(output_file_full_path_dir) del parameters['params_dict']['product_type'] @@ -777,7 +800,9 @@ def test_get_obs_products(self): try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('single_product_retrieval_1.fits') @@ -794,19 +819,28 @@ def test_get_obs_products(self): files_returned = (jwst.get_obs_products( observation_id=observation_id, output_file=output_file_full_path)) - dummyTapHandler.check_call('load_data', parameters) + + args, kwargs = download_mock.call_args + assert kwargs["params"]["RETRIEVAL_TYPE"] == "OBSERVATION" + assert kwargs["params"]["calibrationlevel"] == "ALL" + self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: # self.__remove_folder_contents(folder=output_file_full_path_dir) shutil.rmtree(output_file_full_path_dir) # Test single file zip - output_file_full_path_dir = os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_3" + output_file_full_path_dir = os.path.join( + os.getcwd(), + "temp_test_jwsttap_get_obs_products_3") try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('single_product_retrieval_3.fits.zip') @@ -817,7 +851,9 @@ def test_get_obs_products(self): parameters['output_file'] = output_file_full_path expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval.fits' + extracted_file_1 = os.path.join( + output_file_full_path_dir, + 'single_product_retrieval.fits') expected_files.append(extracted_file_1) try: @@ -826,9 +862,12 @@ def test_get_obs_products(self): cal_level=1, output_file=output_file_full_path)) parameters['params_dict']['calibrationlevel'] = 'LEVEL1ONLY' - dummyTapHandler.check_call('load_data', parameters) + + args, kwargs = download_mock.call_args + assert kwargs["params"]["calibrationlevel"] == "LEVEL1ONLY" self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: # self.__remove_folder_contents(folder=output_file_full_path_dir) shutil.rmtree(output_file_full_path_dir) @@ -836,88 +875,129 @@ def test_get_obs_products(self): parameters['params_dict']['calibrationlevel'] = 'ALL' # Test single file gzip - output_file_full_path_dir = (os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_4") + output_file_full_path_dir = os.path.join( + os.getcwd(), + "temp_test_jwsttap_get_obs_products_4") try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('single_product_retrieval_2.fits.gz') - output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + output_file_full_path = os.path.join( + output_file_full_path_dir, + os.path.basename(file)) shutil.copy(file, output_file_full_path) parameters['output_file'] = output_file_full_path expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval_2.fits.gz' + extracted_file_1 = os.path.join( + output_file_full_path_dir, + 'single_product_retrieval_2.fits.gz') expected_files.append(extracted_file_1) try: files_returned = (jwst.get_obs_products( observation_id=observation_id, output_file=output_file_full_path)) - dummyTapHandler.check_call('load_data', parameters) + + args, kwargs = download_mock.call_args + assert kwargs["params"]["RETRIEVAL_TYPE"] == "OBSERVATION" + assert kwargs["params"]["calibrationlevel"] == "ALL" + self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: # self.__remove_folder_contents(folder=output_file_full_path_dir) shutil.rmtree(output_file_full_path_dir) # Test tar with 3 files, a normal one, a gzip one and a zip one - output_file_full_path_dir = (os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_5") + output_file_full_path_dir = os.path.join( + os.getcwd(), + "temp_test_jwsttap_get_obs_products_5" + ) try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('three_products_retrieval.tar') - output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + output_file_full_path = os.path.join( + output_file_full_path_dir, + os.path.basename(file)) shutil.copy(file, output_file_full_path) parameters['output_file'] = output_file_full_path - expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval_1.fits' - expected_files.append(extracted_file_1) - extracted_file_2 = output_file_full_path_dir + os.sep + 'single_product_retrieval_2.fits.gz' - expected_files.append(extracted_file_2) - extracted_file_3 = output_file_full_path_dir + os.sep + 'single_product_retrieval_3.fits.zip' - expected_files.append(extracted_file_3) + expected_files = [ + os.path.join( + output_file_full_path_dir, + "single_product_retrieval_1.fits"), + os.path.join( + output_file_full_path_dir, + "single_product_retrieval_2.fits.gz"), + os.path.join( + output_file_full_path_dir, + "single_product_retrieval_3.fits.zip"), + ] try: files_returned = (jwst.get_obs_products( observation_id=observation_id, output_file=output_file_full_path)) - dummyTapHandler.check_call('load_data', parameters) + + args, kwargs = download_mock.call_args + assert kwargs["params"]["RETRIEVAL_TYPE"] == "OBSERVATION" + assert kwargs["params"]["calibrationlevel"] == "ALL" + self.__check_extracted_files(files_expected=expected_files, files_returned=files_returned) + get_obs_products_counter += 1 finally: # self.__remove_folder_contents(folder=output_file_full_path_dir) shutil.rmtree(output_file_full_path_dir) + assert download_mock.call_count == get_obs_products_counter + def test_gunzip_file(self): - output_file_full_path_dir = (os.getcwd() + os.sep + "temp_test_jwsttap_gunzip") + output_file_full_path_dir = \ + (os.getcwd() + os.sep + "temp_test_jwsttap_gunzip") try: os.makedirs(output_file_full_path_dir, exist_ok=True) except OSError as err: - print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + print(f"Creation of the directory " + f"{output_file_full_path_dir} failed: " + f"{err.strerror}") raise err file = data_path('single_product_retrieval_2.fits.gz') - output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + output_file_full_path = os.path.join( + output_file_full_path_dir, + os.path.basename(file) + ) shutil.copy(file, output_file_full_path) expected_files = [] - extracted_file_1 = output_file_full_path_dir + os.sep + "single_product_retrieval_2.fits" + extracted_file_1 = os.path.join( + output_file_full_path_dir, + "single_product_retrieval_2.fits" + ) expected_files.append(extracted_file_1) try: extracted_file = (JwstClass.gzip_uncompress_and_rename_single_file( output_file_full_path)) if extracted_file != extracted_file_1: - raise ValueError(f"Extracted file not found: {extracted_file_1}") + raise ValueError(f"Extracted file not found: " + f"{extracted_file_1}") finally: # self.__remove_folder_contents(folder=output_file_full_path_dir) shutil.rmtree(output_file_full_path_dir) @@ -925,13 +1005,20 @@ def test_gunzip_file(self): def __check_results_column(self, results, columnName, description, unit, dataType): c = results[columnName] - assert c.description == description, \ - f"Wrong description for results column '{columnName}'. Expected: '{description}', "\ - f"found '{c.description}'" + assert c.description == description, ( + f"Wrong description for results column '{columnName}'. " + f"Expected: '{description}', found '{c.description}'" + ) assert c.unit == unit, \ - f"Wrong unit for results column '{columnName}'. Expected: '{unit}', found '{c.unit}'" + (f"Wrong unit for results column " + f"'{columnName}'. Expected: " + f"'{unit}', found " + f"'{c.unit}'") assert c.dtype == dataType, \ - f"Wrong dataType for results column '{columnName}'. Expected: '{dataType}', found '{c.dtype}'" + (f"Wrong dataType for results column " + f"'{columnName}'. Expected: " + f"'{dataType}', found " + f"'{c.dtype}'") def __remove_folder_contents(self, folder): for root, dirs, files in os.walk(folder): @@ -954,10 +1041,13 @@ def __check_extracted_files(self, files_expected, files_returned): def test_query_target_error(self): # need to patch simbad query object here - with patch("astroquery.simbad.SimbadClass.query_object", - side_effect=lambda object_name: parse_single_table( - Path(__file__).parent / "data" / f"simbad_{object_name}.vot" - ).to_table()): + with patch( + "astroquery.simbad.SimbadClass.query_object", + side_effect=lambda object_name: parse_single_table( + Path(__file__).parent / "data" / f"simbad_{object_name}" + f".vot" + ).to_table(), + ): jwst = JwstClass(show_messages=False) simbad = SimbadClass() ned = Ned() @@ -965,196 +1055,310 @@ def test_query_target_error(self): # Testing default parameters with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="M1", target_resolver="") - assert "This target resolver is not allowed" in err.value.args[0] + assert ("This target resolver " + "is not allowed") in err.value.args[0] with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target("TEST") - assert ('This target name cannot be determined with this ' - 'resolver: ALL' in err.value.args[0] or 'Failed to parse' in err.value.args[0]) + assert any( + msg in err.value.args[0] + for msg in ( + 'This target name cannot be determined ' + 'with this resolver: ALL', + 'Failed to parse') + ) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="M1", target_resolver="ALL") - assert err.value.args[0] in ["This target name cannot be determined " - "with this resolver: ALL", "Missing " - "required argument: 'width'"] + assert err.value.args[0] in ["This target name cannot be " + "determined with this resolver: " + "ALL", + "Missing required argument: " + "'width'"] # Testing no valid coordinates from resolvers - simbad_file = data_path('test_query_by_target_name_simbad_ned_error.vot') + simbad_file = data_path( + 'test_query_by_target_name_simbad_ned_error.vot') simbad_table = Table.read(simbad_file) simbad.query_object = MagicMock(return_value=simbad_table) - ned_file = data_path('test_query_by_target_name_simbad_ned_error.vot') + ned_file = data_path( + 'test_query_by_target_name_simbad_ned_error.vot') ned_table = Table.read(ned_file) ned.query_object = MagicMock(return_value=ned_table) - vizier_file = data_path('test_query_by_target_name_vizier_error.vot') + vizier_file = data_path( + 'test_query_by_target_name_vizier_error.vot') vizier_table = Table.read(vizier_file) vizier.query_object = MagicMock(return_value=vizier_table) - # coordinate_error = 'coordinate must be either a string or astropy.coordinates' + # coordinate_error = 'coordinate must be either + # a string or astropy.coordinates' with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="TEST", target_resolver="SIMBAD", radius=units.Quantity(5, units.deg)) - assert ('This target name cannot be determined with this ' - 'resolver: SIMBAD' in err.value.args[0] or 'Failed to parse' in err.value.args[0]) + assert any( + msg in err.value.args[0] + for msg in ( + 'This target name cannot be determined ' + 'with this resolver: SIMBAD', + 'Failed to parse') + ) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="TEST", target_resolver="NED", radius=units.Quantity(5, units.deg)) - assert ('This target name cannot be determined with this ' - 'resolver: NED' in err.value.args[0] or 'Failed to parse' in err.value.args[0]) + assert any( + msg in err.value.args[0] + for msg in ( + 'This target name cannot be determined ' + 'with this resolver: NED', + 'Failed to parse') + ) with pytest.raises((ValueError, TableParseError)) as err: jwst.query_target(target_name="TEST", target_resolver="VIZIER", radius=units.Quantity(5, units.deg)) - assert ('This target name cannot be determined with this resolver: ' - 'VIZIER' in err.value.args[0] or 'Failed to parse' in err.value.args[0]) - - def test_remove_jobs(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - job_list = ['dummyJob'] - parameters = {} - parameters['jobs_list'] = job_list - parameters['verbose'] = False - tap.remove_jobs(jobs_list=job_list) - dummyTapHandler.check_call('remove_jobs', parameters) - - def test_save_results(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - job = 'dummyJob' - parameters = {} - parameters['job'] = job - parameters['verbose'] = False - tap.save_results(job) - dummyTapHandler.check_call('save_results', parameters) - - def test_login(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + assert any( + msg in err.value.args[0] + for msg in ( + "This target name cannot be determined " + "with this resolver: VIZIER", + "Failed to parse", + ) + ) + + @patch.object(JwstClass, 'login') + def test_login(self, login_mock): + tap = JwstClass(show_messages=False) parameters = {} parameters['user'] = 'test_user' parameters['password'] = 'test_password' parameters['credentials_file'] = None parameters['verbose'] = False tap.login(user='test_user', password='test_password') - dummyTapHandler.check_call('login', parameters) - - def test_logout(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + login_mock.assert_called_once_with( + user="test_user", + password="test_password" + ) + + @patch.object(JwstClass, 'logout') + def test_logout(self, logout_mock): + tap = JwstClass(show_messages=False) parameters = {} parameters['verbose'] = False tap.logout() - dummyTapHandler.check_call('logout', parameters) + logout_mock.assert_called_once() - def test_set_token_ok(self): + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_set_token_ok(self, mock_execute_servlet_request): old_stdout = sys.stdout # Memorize the default stdout stream sys.stdout = buffer = io.StringIO() - connHandler = DummyConnHandler() - response = DummyResponse(200) - response.set_data(method='GET', body='OK') + tap = JwstClass(show_messages=False) token = 'test_token' - connHandler.set_response(f"{conf.JWST_TOKEN}?token={token}", response) - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) + + # Mock a successful servlet response + mock_execute_servlet_request.return_value = SimpleNamespace(status=200) tap.set_token(token=token) sys.stdout = old_stdout assert ('MAST token has been set successfully' in buffer.getvalue()) - - def test_set_token_anonymous_error(self): + mock_execute_servlet_request.assert_called_once_with( + tap=tap.tap, + query_params={'token': token}, + url=conf.JWST_DOMAIN_SERVER + conf.JWST_TARGET_ACTION, + ) + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_set_token_anonymous_error(self, mock_execute_servlet_request): old_stdout = sys.stdout # Memorize the default stdout stream sys.stdout = buffer = io.StringIO() - connHandler = DummyConnHandler() - response = DummyResponse(403) - response.set_data(method='GET', body='OK') + tap = JwstClass(show_messages=False) token = 'test_token' - connHandler.set_response(f"{conf.JWST_TOKEN}?token={token}", response) - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) + + # Mock 403 error + mock_execute_servlet_request.return_value = SimpleNamespace(status=403) tap.set_token(token=token) sys.stdout = old_stdout - assert ('ERROR: MAST tokens cannot be assigned or requested by anonymous users' in buffer.getvalue()) - - def test_set_token_server_error(self): + assert ('ERROR: MAST tokens cannot be assigned ' + 'or requested by anonymous users' + in buffer.getvalue()) + mock_execute_servlet_request.assert_called_once_with( + tap=tap.tap, + query_params={'token': token}, + url=conf.JWST_DOMAIN_SERVER + conf.JWST_TARGET_ACTION, + ) + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_set_token_server_error(self, mock_execute_servlet_request): old_stdout = sys.stdout # Memorize the default stdout stream sys.stdout = buffer = io.StringIO() - connHandler = DummyConnHandler() - response = DummyResponse(500) - response.set_data(method='GET', body='OK') + tap = JwstClass(show_messages=False) token = 'test_token' - connHandler.set_response(f"{conf.JWST_TOKEN}?token={token}", response) - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) + + # Mock 500 error + mock_execute_servlet_request.return_value = SimpleNamespace(status=500) tap.set_token(token=token) sys.stdout = old_stdout assert ('ERROR: Server error when setting the token' in buffer.getvalue()) - - def test_get_messages_ok(self): + mock_execute_servlet_request.assert_called_once_with( + tap=tap.tap, + query_params={'token': token}, + url=conf.JWST_DOMAIN_SERVER + conf.JWST_TARGET_ACTION, + ) + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + def test_get_messages_ok(self, mock_execute_servlet_request): old_stdout = sys.stdout # Memorize the default stdout stream sys.stdout = buffer = io.StringIO() - connHandler = DummyConnHandler() - response = DummyResponse(200) - response.set_data(method='GET', body='message=SERVER OK') - connHandler.set_response(f"{conf.JWST_MESSAGES}", response) - tapplus = TapPlus(url="http://test:1111/tap", connhandler=connHandler) - tap = JwstClass(tap_plus_handler=tapplus, show_messages=False) + jwst = JwstClass(show_messages=False) + + # Fake response object for parse_messages_response + class FakeResponse: + def iter_lines(self): + return [b"message=SERVER OK"] + + # Calls the parser method in execute_servlet_request + # using the fake response + mock_execute_servlet_request.side_effect = ( + lambda *args, **kwargs: kwargs["parser_method"](FakeResponse()) + ) + + jwst.get_status_messages() - tap.get_status_messages() sys.stdout = old_stdout assert ('SERVER OK' in buffer.getvalue()) + mock_execute_servlet_request.assert_called_once() @pytest.mark.noautofixt - def test_query_get_product(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch("astroquery.esa.utils.utils.EsaTap.query_tap") + def test_query_get_product(self, mock_query_tap): + tap = JwstClass(show_messages=False) file = 'test_file' parameters = {} - parameters['query'] = f"select * from jwst.artifact a where a.filename = '{file}'" - parameters['name'] = None + parameters['query'] = (f"select * from jwst.artifact a " + f"where a.filename = '{file}'") parameters['output_file'] = None parameters['output_format'] = 'votable' parameters['verbose'] = False - parameters['dump_to_file'] = False - parameters['upload_resource'] = None - parameters['upload_table_name'] = None + + # Mock return value for job.get_results() + mock_job = MagicMock() + mock_job.get_results.side_effect = [ + {'artifactid': ['artifact123']}, + {"filename": ["file456.fits"]} + ] + mock_query_tap.return_value = mock_job + tap._query_get_product(file_name=file) - dummyTapHandler.check_call('launch_job', parameters) + + mock_query_tap.assert_called_once_with( + query=f"select * from jwst.artifact a " + f"where a.filename = '{file}'") + + mock_query_tap.reset_mock() artifact = 'test_artifact' - parameters['query'] = f"select * from jwst.artifact a where a.artifactid = '{artifact}'" + parameters['query'] = (f"select * from jwst.artifact a " + f"where a.artifactid = '{artifact}'") tap._query_get_product(artifact_id=artifact) - dummyTapHandler.check_call('launch_job', parameters) - - def test_get_related_observations(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) + mock_query_tap.assert_called_once_with( + query=f"select * from jwst.artifact a " + f"where a.artifactid = '{artifact}'") + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch("astroquery.esa.utils.utils.EsaTap.query_tap") + def test_get_related_observations(self, mock_query_tap): + tap = JwstClass(show_messages=False) obs = 'dummyObs' - tap.get_related_observations(observation_id=obs) - parameters = {} - parameters['query'] = f"select * from jwst.main m where m.members like '%{obs}%'" - parameters['name'] = None - parameters['output_file'] = None - parameters['output_format'] = 'votable' - parameters['verbose'] = False - parameters['dump_to_file'] = False - parameters['upload_resource'] = None - parameters['upload_table_name'] = None - dummyTapHandler.check_call('launch_job', parameters) - - def test_load_async_job(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - tap.load_async_job(jobid=101222) - parameters = {} - parameters['jobid'] = 101222 - parameters['name'] = None - parameters['verbose'] = False - dummyTapHandler.check_call('load_async_job', parameters) + + mock_query_tap.return_value = { + "observationid": ["OBS1", "OBS2"] + } + + result = tap.get_related_observations(observation_id=obs) + assert result == ["OBS1", "OBS2"] + mock_query_tap.assert_called_once() + + mock_query_tap.reset_mock() + mock_query_tap.return_value = Table({"observationid": [""], "members": ["caom:JWST/123 456 789"]}) + + result = tap.get_related_observations(observation_id=obs) + assert result == ["123", "456", "789"] + + assert mock_query_tap.call_args_list[0].kwargs == { + "query": f"select * from {conf.JWST_MAIN_TABLE} m " + f"where m.members like '%{obs}%'" + } + + assert mock_query_tap.call_args_list[1].kwargs == { + "query": f"select m.members from {conf.JWST_MAIN_TABLE} m " + f"where m.observationid='{obs}'" + } + + def test_parse_messages_response(self): + jwst = JwstClass(show_messages=False) + response = Response() + response.status_code = 200 + plain_text = ( + "notification_id1[type: type1,subtype1]=msg1\n" + "notification_id2[type: type2,subtype2]=msg2\n" + "notification_idn[type: typen,subtypen]=msgn" + ) + response._content = plain_text.encode('utf-8') + response.headers['Content-Type'] = 'text/plain' + response.raw = io.BytesIO(response._content) + messages = jwst.parse_messages_response(response) + assert len(messages) == 3 + assert messages == ["msg1", "msg2", "msgn"] + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + @patch.object(JwstClass, 'upload_table') + def test_launch_job_query_and_upload(self, upload_table_mock, mock_query_tap): + jwst = JwstClass(show_messages=False) + + mock_query_tap.return_value = Table({"ID": [1, 2, 3]}) + + jwst.launch_job( + query="SELECT * FROM table", + upload_table_name="my_table", + async_job=False + ) + + mock_query_tap.assert_called_once() + upload_table_mock.assert_called_once() + + args, kwargs = upload_table_mock.call_args + assert kwargs["table_name"] == "my_table" + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch.object(JwstClass, 'query_tap') + @patch.object(JwstClass, 'upload_table') + def test_launch_job_upload_resource_only(self, upload_table_mock, mock_query_tap): + + jwst = JwstClass(show_messages=False) + jwst.launch_job( + query="", + upload_resource="/fake/file.vot", + upload_table_name="my_table" + ) + + upload_table_mock.assert_called_once_with( + conf.JWST_UPLOAD, + upload_resource="/fake/file.vot", + table_name="my_table", + verbose=False + ) + + mock_query_tap.assert_not_called() diff --git a/astroquery/esa/plato/tests/setup_package.py b/astroquery/esa/plato/tests/setup_package.py new file mode 100644 index 0000000000..3621b0cb60 --- /dev/null +++ b/astroquery/esa/plato/tests/setup_package.py @@ -0,0 +1,18 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os + + +# setup paths to the test data +# can specify a single file or a list of files +def get_package_data(): + paths = [os.path.join('data', '*.vot'), + os.path.join('data', '*.xml'), + os.path.join('data', '*.zip'), + os.path.join('data', '*.gz'), + os.path.join('data', '*.tar'), + os.path.join('data', '*.fits'), + os.path.join('data', '*.txt'), + ] # etc, add other extensions + # you can also enlist files individually by names + # finally construct and return a dict for the sub module + return {'astroquery.esa.plato.tests': paths} diff --git a/astroquery/esa/utils/tests/test_utils.py b/astroquery/esa/utils/tests/test_utils.py index 43d41b590d..077b14af74 100644 --- a/astroquery/esa/utils/tests/test_utils.py +++ b/astroquery/esa/utils/tests/test_utils.py @@ -95,6 +95,7 @@ class DummyTapClass(EsaTap): TAP_URL = "dummyUrl" LOGIN_URL = "dummyLogin" LOGOUT_URL = "dummyLogout" + UPLOAD_URL = None class TestEsaUtils: @@ -170,6 +171,30 @@ def test_execute_servlet_request_error(self, mock_tap): tap=mock_tap, query_params=query_params) assert error_message in err.value.args[0] + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService') + def test_execute_servlet_request_post(self, mock_tap): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"ok": True} + mock_tap._session.post.return_value = mock_response + + esautils.execute_servlet_request( + url="https://dummyurl.com/upload", + tap=mock_tap, + method="POST", + query_params={"A": 1}, + data={"TABLE_NAME": "tbl"}, + files={"FILE": ("file.dat", b"DATA")} + ) + + mock_tap._session.post.assert_called_once_with( + url="https://dummyurl.com/upload", + params={"A": 1, "TAPCLIENT": "ASTROQUERY"}, + data={"TABLE_NAME": "tbl"}, + files={"FILE": ("file.dat", b"DATA")} + ) + @patch("pyvo.auth.authsession.AuthSession.get") def test_download_local(self, mock_get, tmp_cwd): iter_content_mock = get_iter_content_mock() @@ -497,3 +522,32 @@ def test_get_metadata(self, table_mock): # Validate first row assert meta[0]["Column"] == "ra" assert meta[0]["Description"] == "Right Ascension" + + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService.capabilities', []) + @patch('astroquery.esa.utils.utils.execute_servlet_request') + @patch('astroquery.esa.utils.utils.pyvo.dal.TAPService') + def test_upload_table_path(self, mock_tapservice, mock_exec): + mock_tap = Mock() + mock_tap._session.post.return_value = Mock(status_code=200) + mock_tapservice.return_value = mock_tap + + tap = DummyTapClass() + tap.UPLOAD_URL = "https://dummyurl.com/Upload" + + mock_exec.return_value = Mock() + + with patch("builtins.open", create=True) as mock_open: + mock_open.return_value = Mock() + + tap.upload_table( + tap.UPLOAD_URL, + upload_resource="dummy.file", + table_name="mytable" + ) + + mock_exec.assert_called_once() + _, kwargs = mock_exec.call_args + + assert kwargs["method"] == "POST" + assert kwargs["data"] == {"TABLE_NAME": "mytable"} + assert "FILE" in kwargs["files"] diff --git a/astroquery/esa/utils/utils.py b/astroquery/esa/utils/utils.py index ff3c4559b0..4cbc1726ab 100644 --- a/astroquery/esa/utils/utils.py +++ b/astroquery/esa/utils/utils.py @@ -135,7 +135,7 @@ def _request(self, method, url, *args, **kwargs): """ # Add the custom query parameter to the URL - additional_params = {'TAPCLIENT': 'ASTROQUERY'} + additional_params = {'TAPCLIENT': 'ASTROQUERY'} if '/login' not in url else {} # Merge the default parameters with the additional request parameters additional_params = additional_params | self.request_parameters if kwargs is not None and 'params' in kwargs: @@ -167,6 +167,7 @@ class EsaTap(BaseVOQuery, BaseQuery): TIMEOUT = 60 REQUEST_PARAMETERS = {} # Additional parameters to be added to all requests THRESHOLD = 1e-5 + UPLOAD_URL = None # missions override if they support uploads """ Class to init ESA TAP Module to communicate with {ESA_ARCHIVE_NAME} Science Archive @@ -392,7 +393,8 @@ def logout(self): """ self._auth_session.logout(logout_url=self.LOGOUT_URL) - def query_tap(self, query, *, async_job=False, output_file=None, output_format='votable', verbose=False): + def query_tap(self, query, *, async_job=False, output_file=None, output_format='votable', + verbose=False): """ Launches a synchronous or asynchronous job to query the {ESA_ARCHIVE_NAME} TAP @@ -410,11 +412,17 @@ def query_tap(self, query, *, async_job=False, output_file=None, output_format=' results format verbose: bool, optional, default False To log the query when executing this method. + upload_resource: str, optional, default None + resource to be uploaded to UPLOAD_SCHEMA + upload_table_name: str, required if uploadResource is provided + Default None + resource temporary table name associated to the uploaded resource Returns ------- An astropy.table object containing the results """ + if async_job: query_result = self.tap.run_async(query) result = query_result.to_table() @@ -641,6 +649,66 @@ def __create_multiple_criteria(self, column, value_list): max_value_clause = self.__create_number_criteria(column, value_list[1], "<=") return f"({min_value_clause} AND {max_value_clause})" + def upload_table(self, + upload_url, + upload_resource, + table_name, + verbose=False): + """ + JWST-specific table upload. Uses the authenticated TAP session. + + """ + + if self._auth_session is None: + raise RuntimeError( + "You must login() before calling upload_table()." + ) + + if upload_resource is None: + raise ValueError("upload_resource must be provided") + if table_name is None: + raise ValueError("table_name must be provided") + + # Prepare payload + payload = {"TABLE_NAME": table_name} + + # Prepare FILE + if hasattr(upload_resource, "read"): + # File-like object + content = upload_resource.read() + if isinstance(content, str): + content = content.encode("utf-8") + files = {"FILE": ("upload_file", content)} + close_needed = False + else: + files = {"FILE": open(upload_resource, "rb")} + close_needed = True + + response = None + + try: + # Use the JWST upload servlet (POST), authenticated TAP session + response = execute_servlet_request( + upload_url, + tap=self.tap, + method="POST", + data=payload, + files=files + ) + + except Exception as e: + if verbose: + print("Exception: ", e) + + finally: + if close_needed: + files["FILE"].close() + + if verbose: + print(f"Uploaded table '{table_name}' to {upload_url}") + + return response + def get_degree_radius(radius): """ @@ -681,7 +749,7 @@ def download_table(astropy_table, output_file=None, output_format=None): astropy_table.write(output_file, format=output_format, overwrite=True) -def execute_servlet_request(url, tap, *, query_params=None, parser_method=None): +def execute_servlet_request(url, tap, *, method='GET', query_params=None, data=None, files=None, parser_method=None): """ Method to execute requests to the servlets on a server @@ -700,15 +768,37 @@ def execute_servlet_request(url, tap, *, query_params=None, parser_method=None): ------- The request with the modified url """ + if query_params is None: + query_params = {} - if 'TAPCLIENT' not in query_params: + if 'TAPCLIENT' not in query_params and '/login' not in url: query_params['TAPCLIENT'] = 'ASTROQUERY' - # Use the TAPService session to perform a custom GET request - response = tap._session.get(url=url, params=query_params) + session = tap._session + + method = method.upper() + + if method == "GET": + # Use the TAPService session to perform a custom GET request + response = session.get(url=url, params=query_params) + + elif method == "POST": + response = session.post( + url=url, + params=query_params, + data=data, + files=files + ) + + else: + raise ValueError(f"Unsupported servlet method: {method}") + if response.status_code == 200: if parser_method is None: - return response.json() + try: + return response.json() + except ValueError: + return response.text else: return parser_method(response) else: diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index 8e6535aa15..f696cb286b 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -99,23 +99,17 @@ service degradation. >>> from astroquery.esa.jwst import Jwst >>> >>> coord = SkyCoord(ra=53, dec=-27, unit=(u.degree, u.degree), frame='icrs') - >>> width = u.Quantity(5, u.deg) - >>> height = u.Quantity(5, u.deg) + >>> width = u.Quantity(1, u.deg) + >>> height = u.Quantity(1, u.deg) >>> result = Jwst.query_region(coordinate=coord, width=width, height=height) >>> result # doctest: +IGNORE_OUTPUT - dist observationid ... - ------------------ ------------------------------ ... - 0.5520678701664351 jw02516010001_xx107_00007_miri ... - 0.5520678701664351 jw02516010001_xx104_00004_miri ... - 0.5520678701664351 jw02516010001_xx102_00002_miri ... - 0.5520678701664351 jw02516010001_xx10b_00011_miri ... - 0.5520678701664351 jw02516010001_xx109_00009_miri ... - 0.5520678701664351 jw02516010001_xx105_00005_miri ... - 0.5520678701664351 jw02516010001_xx103_00003_miri ... - 0.5520678701664351 jw02516010001_xx10a_00010_miri ... - 0.5520678701664351 jw02516010001_xx101_00001_miri ... - 0.5520678701664351 jw02516010001_xx106_00006_miri ... - ... ... ... + + dist observationid calibrationlevel public dataproducttype instrument_name energy_bandpassname target_name targetposition_coordinates_cval1 targetposition_coordinates_cval2 position_bounds_spoly + deg deg + float64 str256 int32 bool str64 str64 str64 str64 float64 float64 object + ------------------ ------------------------------ ---------------- ------ --------------- --------------- ------------------- ----------- -------------------------------- -------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0.5067292953603689 jw01488027001_01201_00001_nrs2 2 True image NIRSPEC OPAQUE;MIRROR UNKNOWN 53.222453933619164 -27.4665362333739 Polygon 53.20094994600003 -27.504268390999997 53.18004140700001 -27.449039869999986 53.24354427599998 -27.42825785399999 53.26482467600001 -27.484746299999987 + 0.5067292953603689 jw01488027001_01201_00001_nrs2 1 True image NIRSPEC OPAQUE;MIRROR UNKNOWN 53.222453933619164 -27.4665362333739 Polygon 53.20094994600003 -27.504268390999997 53.18004140700001 -27.449039869999986 53.24354427599998 -27.42825785399999 53.26482467600001 -27.484746299999987 1.2. Cone search @@ -128,24 +122,29 @@ service degradation. >>> from astroquery.esa.jwst import Jwst >>> >>> coord = SkyCoord(ra=53, dec=-27, unit=(u.degree, u.degree), frame='icrs') - >>> radius = u.Quantity(5.0, u.deg) - >>> j = Jwst.cone_search(coordinate=coord, radius=radius, async_job=True) - INFO: Query finished. [astroquery.utils.tap.core] - >>> result = j.get_results() + >>> radius = u.Quantity(1.0, u.deg) + >>> result = Jwst.cone_search(coordinate=coord, radius=radius, async_job=True) >>> result # doctest: +IGNORE_OUTPUT - dist observationid ... - ------------------ ------------------------------ ... - 0.5520678701664351 jw02516010001_xx107_00007_miri ... - 0.5520678701664351 jw02516010001_xx104_00004_miri ... - 0.5520678701664351 jw02516010001_xx102_00002_miri ... - 0.5520678701664351 jw02516010001_xx10b_00011_miri ... - 0.5520678701664351 jw02516010001_xx109_00009_miri ... - 0.5520678701664351 jw02516010001_xx105_00005_miri ... - 0.5520678701664351 jw02516010001_xx103_00003_miri ... - 0.5520678701664351 jw02516010001_xx10a_00010_miri ... - 0.5520678701664351 jw02516010001_xx101_00001_miri ... - 0.5520678701664351 jw02516010001_xx106_00006_miri ... - ... ... ... +
+ observationid calibrationlevel public dataproducttype ... target_name targetposition_coordinates_cval1 targetposition_coordinates_cval2 position_bounds_spoly + ... deg deg + str256 int32 bool str64 ... str64 float64 float64 object + -------------------------------------------- ---------------- ------ --------------- ... ------------------------------ -------------------------------- -------------------------------- ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + hlsp_jems_jwst_niriss_goods-s_f430m_v1.0_drz 4 True image ... GOODS-South 53.13978654458929 -27.900186014457034 Polygon 53.143803 -27.927580000000003 53.109126 -27.903765000000032 53.13575099999998 -27.872764999999983 53.17046199999999 -27.896657000000015 + hlsp_jems_jwst_nircam_goods-s_f210m_v1.0_drz 4 True image ... GOODS-South 53.143638298315224 -27.801618346014052 Polygon 53.096915 -27.824334000000007 53.122494000000025 -27.794125 53.15620000000001 -27.75595300000002 53.190561000000024 -27.778544000000018 53.16450600000003 -27.809449000000026 53.13050300000002 -27.847565000000017 + hlsp_jems_jwst_nircam_goods-s_f480m_v1.0_drz 4 True image ... GOODS-South 53.143437544017296 -27.801695104861977 Polygon 53.09720799999998 -27.824191999999996 53.12226799999999 -27.794775000000005 53.15615300000003 -27.756444000000002 53.189827 -27.77872799999998 53.16437400000002 -27.808874000000024 53.130314999999996 -27.847223999999976 + hlsp_jems_jwst_nircam_goods-s_f182m_v1.0_drz 4 True image ... GOODS-South 53.14363713303235 -27.80161768929551 Polygon 53.096915 -27.824334000000007 53.122494000000025 -27.79411700000001 53.15620000000001 -27.75595300000002 53.190561000000024 -27.778544000000018 53.16450600000003 -27.809449000000026 53.13050300000002 -27.847565000000017 + hlsp_jems_jwst_nircam_goods-s_f430m_v1.0_drz 4 True image ... GOODS-South 53.14343758237035 -27.801708096392474 Polygon 53.09722600000002 -27.82435900000001 53.12227700000001 -27.794765999999985 53.15615300000003 -27.756444000000002 53.189827 -27.77872799999998 53.16437400000002 -27.808874000000024 53.130314999999996 -27.847223999999976 + hlsp_jems_jwst_nircam_goods-s_f460m_v1.0_drz 4 True image ... GOODS-South 53.143440120302536 -27.801707446703716 Polygon 53.09723600000001 -27.824366999999995 53.12227700000001 -27.794775000000005 53.15615300000003 -27.756444000000002 53.189827 -27.77872799999998 53.16437400000002 -27.808865999999973 53.130314999999996 -27.847223999999976 + hlsp_jems_jwst_niriss_goods-s_f480m_v1.0_drz 4 True image ... GOODS-South 53.13978632823965 -27.900183039523228 Polygon 53.143803 -27.927580000000003 53.109126 -27.903765000000032 53.13575099999998 -27.872756000000024 53.17046199999999 -27.896657000000015 + ... ... ... ... ... ... ... ... ... + jw09947005001_xx102_00002_nirspec -1 False spectrum ... PANO_j033224m2756_hzmask 53.05677244086242 -27.991837786812724 Polygon 53.096969928409976 -28.01611837930999 53.02950963081 -28.027253231710016 53.01652523938 -27.96799365591003 53.083769109779986 -27.95602420377999 + jw07208043001_xx102_00002_nirspec -1 False spectrum ... panoramic_j033224m2756_id48361 52.967296645552835 -27.91594920163261 Polygon 52.96726340586999 -27.916570782990007 52.966603101559976 -27.915900119979998 52.967329862730026 -27.915327690959998 52.96799020715001 -27.91599813632998 + jw12577010001_xx108_00001_nirspec -1 False spectrum ... MPT_input_P2_v0.1 53.18150426082802 -27.788964910220372 Polygon 53.14066196510001 -27.81228690420001 53.15511898750003 -27.75306849454997 53.22188803278999 -27.765402558849996 53.208412411089995 -27.824806072489967 + jw12537018001_xx104_00004_nircam -1 False image ... C26202 53.13706305814418 -27.863345866648146 Polygon 53.137841297059985 -27.864035186479992 53.13628697504 -27.864035125519973 53.13628547561999 -27.862656598209984 53.13783826878002 -27.862656105589984 + jw12577008001_xx102_00001_nirspec -1 False spectrum ... MPT_input_P2_v0.1 53.18101100892586 -27.788873798924087 Polygon 53.14016864170001 -27.81219564789 53.15462592045 -27.752977289580016 53.22139485402998 -27.76531159096 53.20791897390998 -27.82471505678 + jw07081002001_xx103_00003_nirspec -1 False spectrum ... Highz-CNO-Targets 53.14603497119161 -27.78278614388954 Polygon 53.125203002309995 -27.82163964418002 53.10224041494998 -27.76456272203999 53.166341358630014 -27.743940829299994 53.19024383501 -27.80071758985997 + jw12340001001_xx107_00003_nirspec -1 False spectrum ... overdensity_members 52.98086594387915 -27.792269083676928 Polygon 53.008495388439975 -27.827651680190023 52.94111846859002 -27.816712507090003 52.953009947299996 -27.757277977119983 53.02052311353999 -27.767369056419994 1.3. Query by target name @@ -164,22 +163,29 @@ element in the list if the target name cannot be resolved). >>> >>> target_name = 'M1' >>> target_resolver = 'ALL' - >>> radius = u.Quantity(5, u.deg) + >>> radius = u.Quantity(1, u.deg) >>> result = Jwst.query_target(target_name=target_name, target_resolver=target_resolver, radius=radius) >>> result # doctest: +IGNORE_OUTPUT - dist observationid ... - -------------------- -------------------------------- ... - 0.003349189664076155 jw01714001004_xx106_00002_miri ... - 0.003349189664076155 jw01714001003_xx10q_00002_miri ... - 0.003349189664076155 jw01714001006_xx107_00003_miri ... - 0.003349189664076155 jw01714001002_xx105_00001_miri ... - 0.003349189664076155 jw01714005001_xx106_00006_nircam ... - 0.003349189664076155 jw01714001003_xx10w_00004_miri ... - 0.003349189664076155 jw01714001002_xx103_00003_miri ... - 0.003349189664076155 jw01714001004_xx104_00004_miri ... - 0.003349189664076155 jw01714001006_xx10n_00003_miri ... - 0.003349189664076155 jw01714001005_xx10p_00001_miri ... - ... ... ... +
+ observationid calibrationlevel public dataproducttype instrument_name energy_bandpassname target_name targetposition_coordinates_cval1 targetposition_coordinates_cval2 position_bounds_spoly + deg deg + str256 int32 bool str64 str64 str64 str64 float64 float64 object + --------------------------------- ---------------- ------ --------------- --------------- ------------------- ---------------------- -------------------------------- -------------------------------- ----------------------------------------------------------------------------------------------------------------------------------------------------------- + jw01714-o005_t003_miri_ch4-long 3 True cube MIRI/IFU CH4-LONG CRAB-NEBULA-FIL2 83.64303271855034 21.99404662161963 Polygon 83.64156476099977 21.992782729999995 83.64156476099977 21.995310508000028 83.64450067599985 21.995310508000028 83.64450067599985 21.992782729999995 + jw01714-o003_t004_miri_ch3-short 3 True cube MIRI/IFU CH3-SHORT CRAB-NEBULA-BKG 83.65960468044797 21.931696022675624 Polygon 83.65840687400018 21.93041824399999 83.65840687400018 21.932973799999985 83.66080248699998 21.932973799999985 83.66080248699998 21.93041824399999 + jw01714-c1004_t003_miri_ch1-short 3 True cube MIRI/IFU CH1-SHORT CRAB-NEBULA-FIL2 83.64319158176133 21.99416787269692 Polygon 83.64241266900012 21.993553982000016 83.64241266900012 21.994781759999984 83.64397049499992 21.994781759999984 83.64397049499992 21.993553982000016 + jw01714-o004_t002_miri_ch4-short 3 True cube MIRI/IFU CH4-SHORT CRAB-NEBULA-FIL1 83.62295758403198 22.00812603007894 Polygon 83.62148947999995 22.006764917000023 83.62148947999995 22.00948713900001 83.62442568799975 22.00948713900001 83.62442568799975 22.006764917000023 + jw01714-o002_t004_miri_f1800w 3 True image MIRI/IMAGE F1800W CRAB-NEBULA-BKG 83.6588126140977 21.926351504834074 Polygon 83.63997091399993 21.945036449000018 83.6788455260002 21.943937831999982 83.67764936799988 21.907664414999974 83.63878465200001 21.90876275200001 + jw01714-o005_t003_miri_ch2-short 3 True cube MIRI/IFU CH2-SHORT CRAB-NEBULA-FIL2 83.64310713903527 21.994186937886155 Polygon 83.64213948799977 21.993431380999976 83.64213948799977 21.99494249200003 83.64407478999975 21.99494249200003 83.64407478999975 21.993431380999976 + jw01714-o003_t004_miri_ch4-medium 3 True cube MIRI/IFU CH4-MEDIUM CRAB-NEBULA-BKG 83.65944519557817 21.93143373136718 Polygon 83.65797788100006 21.929780953999995 83.65797788100006 21.933086509000006 83.66091250999979 21.933086509000006 83.66091250999979 21.929780953999995 + ... ... ... ... ... ... ... ... ... ... + jw11816001001_xx109_00001_miri -1 False spectrum MIRI/IFU Crab-Nebula-K1 83.64300529853574 21.994736084514088 Polygon 83.64261143469008 21.995312661570015 83.64356249058979 21.995182077500022 83.64339915486977 21.994159494679984 83.64244810567027 21.994290105590014 + jw11816010001_xx106_00002_nirspec -1 False spectrum NIRSPEC/IFU F170LP;G235H Crab-Nebula-D4 83.6219308810129 22.009298019320624 Polygon 83.62189920105011 22.008676439150005 83.62126986743026 22.00934709850002 83.62196253927003 22.009919531240005 83.62259190943027 22.00924908343002 + jw11816001001_xx103_00003_miri -1 False spectrum MIRI/IFU Crab-Nebula-K1 83.6429201362943 21.994747927344747 Polygon 83.64252627217027 21.995324503870016 83.6434773282298 21.995193920290014 83.64331399311028 21.994171337389993 83.64236294376008 21.994301947810026 + jw11816006001_xx108_00004_miri -1 False spectrum MIRI/IFU Crab-Nebula-Background 83.65924187725976 21.931406383779958 Polygon 83.65884818702011 21.931982960399996 83.65979881947018 21.931852378789998 83.65963555962011 21.93082979554998 83.6586849338599 21.93096040400001 + jw11816011001_xx103_00003_nirspec -1 False spectrum NIRSPEC/IFU F170LP;G235H Crab-Nebula-A1 83.64065363300976 21.990259497183786 Polygon 83.64062195736022 21.989637915509988 83.63999270868021 21.99030857523002 83.64068528793976 21.990881007559995 83.64131457314983 21.990210559379996 + jw11816008001_xx101_00001_nirspec -1 False spectrum NIRSPEC/IFU F170LP;G235H Crab-Nebula-B1 83.64314773390427 21.995031159483062 Polygon 83.64311605687985 21.99440957827 83.64248678721012 21.99508023813 83.64317938990992 21.99565267030998 83.64380869612005 21.994982221990007 + jw11816009001_xx103_00003_nirspec -1 False spectrum NIRSPEC/IFU F170LP;G235H Crab-Nebula-B2 83.63352863412126 21.994953941120784 Polygon 83.63349695726022 21.994332359949993 83.63286768776007 21.995003019670023 83.6335602899402 21.995575452 83.63418959596986 21.99490500382 This method uses the same parameters as query region, but also includes the target name and the catalogue @@ -192,25 +198,31 @@ This method uses the same parameters as query region, but also includes the targ >>> >>> target_name = 'LMC' >>> target_resolver = 'NED' - >>> width = u.Quantity(5, u.deg) - >>> height = u.Quantity(5, u.deg) + >>> width = u.Quantity(1, u.deg) + >>> height = u.Quantity(1, u.deg) >>> result = Jwst.query_target(target_name=target_name, target_resolver=target_resolver, ... width=width, height=height, async_job=True) - INFO: Query finished. [astroquery.utils.tap.core] >>> result # doctest: +IGNORE_OUTPUT - dist observationid ... - ------------------- ---------------------------------- ... - 0.25984680687093176 jw01043010001_02101_00013_mirimage ... - 0.25984680687093176 jw01043010001_02101_00004_mirimage ... - 0.25984680687093176 jw01043010001_02101_00014_mirimage ... - 0.25984680687093176 jw01043007001_02101_00006_mirimage ... - 0.25984680687093176 jw01043008001_02101_00016_mirimage ... - 0.25984680687093176 jw01043007001_02101_00008_mirimage ... - 0.25984680687093176 jw01043009001_02101_00013_mirimage ... - 0.25984680687093176 jw01043007001_02101_00007_mirimage ... - 0.25984680687093176 jw01043007001_02101_00009_mirimage ... - 0.25984680687093176 jw01043009001_02101_00004_mirimage ... - ... ... ... +
+ dist observationid calibrationlevel public dataproducttype instrument_name energy_bandpassname target_name targetposition_coordinates_cval1 targetposition_coordinates_cval2 position_bounds_spoly + deg deg + float64 str256 int32 bool str64 str64 str64 str64 float64 float64 object + ---------------------- ------------------------------------- ---------------- ------ --------------- --------------- ------------------- ----------- -------------------------------- -------------------------------- ----------------------------------------------------------------------------------------------------------------------------------------------------------- + 1.2807763824776493e-06 jw01518023001_03108_00001_mirifushort 2 True image MIRI/IFU CH12-LONG NAME-LMC 80.894167644957 -69.75610901189543 Polygon 80.89503120600004 -69.75672262899971 80.8925404189998 -69.75649476599992 80.89330414900002 -69.7554953780002 80.89579481800017 -69.75572325800023 + 1.2807763824776493e-06 jw01518023001_03108_00001_mirifushort 1 True image MIRI/IFU CH12-LONG NAME-LMC 80.894167644957 -69.75610901189543 Polygon 80.89503120600004 -69.75672262899971 80.8925404189998 -69.75649476599992 80.89330414900002 -69.7554953780002 80.89579481800017 -69.75572325800023 + 1.2810293979654214e-06 jw01518023001_03108_00001_mirimage 2 True image MIRI/IMAGE F1500W NAME-LMC 80.894167644241 -69.7561090117718 Polygon 80.89503120600004 -69.75672262899971 80.892540418 -69.75649476599992 80.89330414800021 -69.7554953780002 80.89579481699978 -69.75572325800023 + 1.2810293979654214e-06 jw01518023001_03108_00001_mirimage 1 True image MIRI/IMAGE F1500W NAME-LMC 80.894167644241 -69.7561090117718 Polygon 80.89503120600004 -69.75672262899971 80.892540418 -69.75649476599992 80.89330414800021 -69.7554953780002 80.89579481699978 -69.75572325800023 + 1.2826137646056745e-06 jw01518023001_03108_00001_mirifulong 1 True image MIRI/IFU CH34-LONG NAME-LMC 80.89416764521805 -69.75610900944068 Polygon 80.89503120600004 -69.7567226279999 80.892540418 -69.7564947650001 80.89330414800021 -69.7554953780002 80.89579481699978 -69.75572325699984 + 1.2826137646056745e-06 jw01518023001_03108_00001_mirifulong 2 True image MIRI/IFU CH34-LONG NAME-LMC 80.89416764521805 -69.75610900944068 Polygon 80.89503120600004 -69.7567226279999 80.892540418 -69.7564947650001 80.89330414800021 -69.7554953780002 80.89579481699978 -69.75572325699984 + 1.2934410724581988e-06 jw01518023001_03107_00001_mirifushort 2 True image MIRI/IFU CH12-MEDIUM NAME-LMC 80.89416771102887 -69.75610897741043 Polygon 80.89503127300001 -69.7567225930002 80.89254048599977 -69.75649472999983 80.89330421500017 -69.75549534299994 80.89579488399976 -69.75572322200014 + ... ... ... ... ... ... ... ... ... ... ... + 0.4265750051094012 jw01090001001_06201_00001_mirimage 1 True image MIRI/IMAGE F2550W SMP-LMC-58 81.2551520390941 -70.16435805966589 Polygon 81.29686033499998 -70.14711260200019 81.30591980500023 -70.17829367299973 81.21377956800012 -70.18160151199997 81.2040258179999 -70.15049733599994 + 0.4265896619279505 jw01090001001_38201_00001_mirimage 1 True image MIRI/IMAGE F560W SMP-LMC-58 81.25516290359896 -70.16437225938321 Polygon 81.29687138799986 -70.14712664300018 81.30593053399993 -70.17830808600002 81.21379077699977 -70.18161594000028 81.20403638600017 -70.15051135800007 + 0.4265896619279505 jw01090001001_38201_00001_mirimage 2 True image MIRI/IMAGE F560W SMP-LMC-58 81.25516290359896 -70.16437225938321 Polygon 81.29687138799986 -70.14712664300018 81.30593053399993 -70.17830808600002 81.21379077699977 -70.18161594000028 81.20403638600017 -70.15051135800007 + 0.4265898346276089 jw01090001001_39201_00001_mirimage 1 True image MIRI/IMAGE F560W SMP-LMC-58 81.25516209090127 -70.16437252444791 Polygon 81.29687060799975 -70.14712691700005 81.30592969700025 -70.17830836100026 81.21378993299972 -70.18161619699991 81.20403559799986 -70.15051161199972 + 0.4265898346276089 jw01090001001_39201_00001_mirimage 2 True image MIRI/IMAGE F560W SMP-LMC-58 81.25516209090127 -70.16437252444791 Polygon 81.29687060799975 -70.14712691700005 81.30592969700025 -70.17830836100026 81.21378993299972 -70.18161619699991 81.20403559799986 -70.15051161199972 + 0.4732768254894247 jw04472009001_01201_00001_nis 2 True image NIRISS/IMAGE F380M UNKNOWN 81.33835678371165 -70.20429529782012 Polygon 81.39036938199988 -70.18477676099992 81.396616906 -70.22186495400021 81.28627985299987 -70.22379789700001 81.2801540589998 -70.18671650800016 + 0.4732768254894247 jw04472009001_01201_00001_nis 1 True image NIRISS/IMAGE F380M UNKNOWN 81.33835678371165 -70.20429529782012 Polygon 81.39036938199988 -70.18477676099992 81.396616906 -70.22186495400021 81.28627985299987 -70.22379789700001 81.2801540589998 -70.18671650800016 1.4 Getting data products @@ -223,14 +235,26 @@ To query the data products associated with a certain Observation ID >>> from astroquery.esa.jwst import Jwst >>> product_list = Jwst.get_product_list(observation_id='jw01063107001_02101_00013_nrca3') >>> print(product_list['filename']) # doctest: +IGNORE_OUTPUT - filename - ------------------------------------------------ - jw01063-o107_20220328t013707_image2_144_asn.json - jw01063107001_02101_00013_nrca3_cal.fits - jw01063107001_02101_00013_nrca3_cal.jpg - jw01063107001_02101_00013_nrca3_cal_thumb.jpg - jw01063107001_02101_00013_nrca3_i2d.fits - ... + filename + ----------------------------------------------------- + jw01063-o107_20230701t234631_image2_00083_asn.json + jw01063107001_02101_00013_nrca3_cal.fits + jw01063107001_02101_00013_nrca3_cal.jpg + jw01063107001_02101_00013_nrca3_cal_thumb.jpg + jw01063107001_02101_00013_nrca3_i2d.fits + jw01063107001_02101_00013_nrca3_o107_crf.fits + jw01063107001_02101_00013_nrca3_o107_crf.jpg + jw01063107001_02101_00013_nrca3_o107_crf_thumb.jpg + ... + jw01063107001_02101_00013_nrca3_rateints_thumb.jpg + jw01063107001_02101_00013_nrca3_trapsfilled.fits + jw01063107001_02101_00013_nrca3_trapsfilled.jpg + jw01063107001_02101_00013_nrca3_trapsfilled_thumb.jpg + jw01063107001_02101_00013_nrca3_uncal.fits + jw01063107001_02101_00013_nrca3_uncal.jpg + jw01063107001_02101_00013_nrca3_uncal_thumb.jpg + jw01063_20230701t234631_pool.csv + Length = 21 rows You can filter by product type and calibration level (using a numerical @@ -242,38 +266,19 @@ all the products associated to this observation_id with the same and lower level >>> from astroquery.esa.jwst import Jwst >>> product_list = Jwst.get_product_list(observation_id='jw01023029001_02101_00004_mirimage', product_type='science') >>> print(product_list['filename']) - filename - ------------------------------------------------- - jw01023029001_02101_00004_mirimage_c1006_crf.fits - jw01023029001_02101_00004_mirimage_cal.fits - jw01023029001_02101_00004_mirimage_i2d.fits - jw01023029001_02101_00004_mirimage_o029_crf.fits - jw01023029001_02101_00004_mirimage_rate.fits - ... + filename + ------------------------------------------------ + jw01023029001_02101_00004_mirimage_cal.fits + jw01023029001_02101_00004_mirimage_i2d.fits + jw01023029001_02101_00004_mirimage_o029_crf.fits + jw01023029001_02101_00004_mirimage_rate.fits + jw01023029001_02101_00004_mirimage_rateints.fits + jw01023029001_02101_00004_mirimage_uncal.fits -To download a data product +To download a data product, once you have the file name: -.. doctest-remote-data:: - - >>> from astroquery.esa.jwst import Jwst - >>> query = """select o.observationid, a.artifactid, a.filename - ... from jwst.observation o join jwst.artifact a on a.obsid = o.obsid - ... where o.proposal_id = '01166' and o.intent = 'science'""" - >>> job = Jwst.launch_job(query, async_job=True) - INFO: Query finished. [astroquery.utils.tap.core] - >>> results = job.get_results() - >>> results # doctest: +IGNORE_OUTPUT - observationid artifactid filename - ------------------------------- ------------------------------------ ---------------------------------------------------- - jw01166091001_02102_00002_nrca3 6ab73824-6587-4bca-84a8-eb48ac7251be jw01166-o091_20220505t044658_wfs-image2_058_asn.json - jw01166091001_02102_00002_nrca3 c999249f-c554-4a5f-bd6a-099f1fe48864 jw01166091001_02102_00002_nrca3_cal.fits - jw01166091001_02102_00002_nrca3 004433de-a4b3-4dc6-9177-0f9d78a97bda jw01166091001_02102_00002_nrca3_cal.jpg - jw01166091001_02102_00002_nrca3 5b89920d-532d-43be-b70b-973c4bfdfdcc jw01166091001_02102_00002_nrca3_cal_thumb.jpg - jw01166091001_02102_00002_nrca3 5a86e351-3066-4c46-8698-c49aade6ec98 jw01166091001_02102_00002_nrca3_rate.fits - ... ... ... - >>> output_file = Jwst.get_product(artifact_id='6ab73824-6587-4bca-84a8-eb48ac7251be') # doctest: +SKIP - >>> output_file = Jwst.get_product(file_name='jw01166091001_02102_00002_nrca3_cal.fits') # doctest: +SKIP + >>> output_file = Jwst.get_product(file_name='jw01023029001_02101_00004_mirimage_uncal.fits') # doctest: +SKIP To download products by observation identifier, it is possible to use the get_obs_products function, with the same parameters @@ -282,16 +287,16 @@ than get_product_list, it also supports product_type parameter as string or list .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> observation_id = 'jw01122001001_0210r_00001_nrs2' - >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type='science') + >>> observation_id = 'jw02739013001_02103_00001_nrcb1' + >>> Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type='science') # doctest: +IGNORE_OUTPUT -Here product_type as list: +A list of products can be retrieved by defining product_type as list: .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> observation_id = 'jw01122001001_0210r_00001_nrs2' + >>> observation_id = 'jw02739013001_02103_00001_nrcb1' >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type=['science', 'preview']) A temporary directory is created with the files and a list of the them is provided. @@ -320,23 +325,14 @@ Using the observation ID as input parameter, this function will retrieve the obs >>> observation_id = 'jw02739-o001_t001_nircam_clear-f444w' >>> results = Jwst.get_related_observations(observation_id=observation_id) >>> results[0:5] - ['jw02739001001_02105_00001_nrcalong', - 'jw02739001001_02105_00001_nrcblong', - 'jw02739001001_02105_00002_nrcalong', - 'jw02739001001_02105_00002_nrcblong', - 'jw02739001001_02105_00003_nrcalong'] + ['jw02739001001_02105_00001_nrcalong', 'jw02739001001_02105_00001_nrcblong', 'jw02739001001_02105_00002_nrcalong', 'jw02739001001_02105_00002_nrcblong', 'jw02739001001_02105_00003_nrcalong'] To query the data products associated with a certain Proposal ID and filtered by product_type. .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> observation_list = Jwst.download_files_from_program(proposal_id='6651', product_type='preview') # doctest: +IGNORE_OUTPUT - INFO: Query finished. [astroquery.utils.tap.core] - INFO: Downloading products for Observation ID: jw06651001001_05201_00001_nis [astroquery.esa.jwst.core] - INFO: Downloading products for Observation ID: jw06651002001_05201_00001_nis [astroquery.esa.jwst.core] - >>> print(observation_list) # doctest: +IGNORE_OUTPUT - ['jw06651001001_05201_00001_nis', 'jw06651002001_05201_00001_nis'] + >>> observation_list = Jwst.download_files_from_program(proposal_id='1172', product_type='preview') # doctest: +IGNORE_OUTPUT 1.5 Getting public tables @@ -347,20 +343,11 @@ To load only table names (TAP+ capability) .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> tables = Jwst.load_tables(only_names=True) - INFO: Retrieving tables... [astroquery.utils.tap.core] - INFO: Parsing tables... [astroquery.utils.tap.core] - INFO: Done. [astroquery.utils.tap.core] - >>> print(*(table.name for table in tables), sep="\n") - ivoa.obscore - jwst.archive - jwst.artifact - jwst.chunk - jwst.harvestskipuri - jwst.main - jwst.moc - jwst.observation + >>> tables = Jwst.get_tables(only_names=True) + >>> tables + ['ivoa.ObsCore', 'jwst.archive', 'jwst.artifact', 'jwst.chunk', 'jwst.harvestskipuri', ... + 'tap_schema.tables'] To load table names (TAP compatible) @@ -368,44 +355,35 @@ To load table names (TAP compatible) .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> tables = Jwst.load_tables() - INFO: Retrieving tables... [astroquery.utils.tap.core] - INFO: Parsing tables... [astroquery.utils.tap.core] - INFO: Done. [astroquery.utils.tap.core] + >>> tables = Jwst.get_tables() >>> print(*(table.name for table in tables), sep="\n") - ivoa.obscore + ivoa.ObsCore jwst.archive jwst.artifact jwst.chunk jwst.harvestskipuri - jwst.main ... + tap_schema.schemas + tap_schema.tables -To load only a table (TAP+ capability) +To load only a table (TAP+ capability), the following method can be used. .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> table = Jwst.load_table('jwst.main') - >>> print(table) # doctest: +SKIP - TAP Table name: jwst.jwst.main - Description: None - Num. columns: 109 - + >>> table = Jwst.get_table('jwst.archive') Once a table is loaded, columns can be inspected .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> table = Jwst.load_table('jwst.main') + >>> table = Jwst.get_table('jwst.archive') >>> print(*(column.name for column in table.columns), sep="\n") # doctest: +IGNORE_OUTPUT - "public" - algorithm_name calibrationlevel collection - creatorid dataproducttype + detector energy_bandpassname ... @@ -427,52 +405,31 @@ Query without saving results in a file: >>> from astroquery.esa.jwst import Jwst >>> - >>> job = Jwst.launch_job("SELECT TOP 100 " + >>> result = Jwst.launch_job("SELECT TOP 100 " ... "instrument_name, proposal_id, calibrationlevel, " ... "dataproducttype " - ... "FROM jwst.main ORDER BY instrument_name, observationuri") - >>> - >>> print(job) # doctest: +IGNORE_OUTPUT -
- name dtype - ---------------- ------ - instrument_name object - proposal_id object - calibrationlevel int32 - dataproducttype object - Jobid: None - Phase: COMPLETED - Owner: None - Output file: 1661441953031O-result.vot.gz - Results: None - >>> result = job.get_results() + ... "FROM jwst.archive ORDER BY instrument_name") >>> result # doctest: +IGNORE_OUTPUT
instrument_name proposal_id calibrationlevel dataproducttype - object object int32 object + str64 str64 int32 str64 --------------- ----------- ---------------- --------------- - FGS 1014 3 image - FGS 1014 3 image - FGS 1014 3 image - FGS 1014 3 image - FGS 1014 3 image - FGS 1014 2 image - FGS 1014 1 image - FGS 1014 1 image - FGS 1014 2 image - FGS 1014 2 image + FGS 1150 2 image + FGS 1141 1 image + FGS 1148 2 image + FGS 1148 2 image + FGS 1148 2 image + FGS 1159 1 image + FGS 1151 1 image ... ... ... ... - FGS 1017 2 image - FGS 1017 1 image - FGS 1017 2 image - FGS 1017 1 image - FGS 1017 2 image - FGS 1017 1 image - FGS 1017 2 image - FGS 1017 1 image - FGS 1017 2 image - FGS 1017 1 image - FGS 1017 2 image + FGS 1148 1 image + FGS 1159 1 image + FGS 1148 1 image + FGS 1155 2 image + FGS 1147 3 image + FGS 1148 2 image + FGS 1158 1 image + FGS 1151 2 image Query saving results in a file: @@ -483,29 +440,8 @@ Query saving results in a file: >>> job = Jwst.launch_job("SELECT TOP 100 " ... "instrument_name, proposal_id, calibrationlevel, " ... "dataproducttype " - ... "FROM jwst.main ORDER BY instrument_name, observationuri", - ... dump_to_file=True) - >>> - >>> print(job) # doctest: +IGNORE_OUTPUT - Jobid: None - Phase: COMPLETED - Owner: None - Output file: 1655978085454O-result.vot.gz - Results: None - >>> result = job.get_results() - >>> result # doctest: +IGNORE_OUTPUT -
- instrument_name proposal_id calibrationlevel dataproducttype - object object int32 object - --------------- ----------- ---------------- --------------- - FGS 01014 3 image - FGS 01014 3 image - FGS 01014 3 image - FGS 01014 3 image - FGS 01014 3 image - FGS 01014 3 image - FGS 01014 2 image - ... ... ... ... + ... "FROM jwst.archive ORDER BY instrument_name", + ... output_file="file.vot") 1.7 Synchronous query on an 'on-the-fly' uploaded table @@ -513,16 +449,13 @@ Query saving results in a file: A table can be uploaded to the server in order to be used in a query. -.. TODO: a local file need to be added for this example .. doctest-skip:: >>> from astroquery.esa.jwst import Jwst >>> upload_resource = 'mytable.xml.gz' - >>> j = Jwst.launch_job(query="SELECT * from tap_upload.table_test", + >>> result = Jwst.launch_job(query="SELECT * from tap_upload.table_test", ... upload_resource=upload_resource, ... upload_table_name="table_test", verbose=True) - >>> result = j.get_results() - >>> result.pprint() source_id alpha delta --------- ----- ----- a 1.0 2.0 @@ -542,46 +475,35 @@ Query without saving results in a file: .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> job = Jwst.launch_job("select top 100 * from jwst.main", async_job=True) - INFO: Query finished. [astroquery.utils.tap.core] - >>> print(job) # doctest: +IGNORE_OUTPUT - Jobid: 1542383562372I - Phase: COMPLETED - Owner: None - Output file: async_20181116165244.vot - Results: None - >>> r = job.get_results() - >>> r['observationid'] # doctest: +IGNORE_OUTPUT - - jw01070001002_04101_00003_nrca1 - jw01070001002_04101_00003_nrca1 - jw01286007001_xx10k_00001_nirspec - jw01290023001_xx10b_00001_miri - jw01568006004_xx101_00001_nircam - ... + >>> result = Jwst.launch_job("select top 100 * from jwst.archive", async_job=True) + >>> result +
+ calibrationlevel collection dataproducttype detector energy_bandpassname energy_bounds_lower energy_bounds_upper energy_bounds_width instrument_name intent ... target_name targetposition_coordinates_cval1 targetposition_coordinates_cval2 template time_bounds_lower time_bounds_upper time_exposure vis_cube vis_image wave_central + m m m ... deg deg d d s µm + int32 str64 str64 object str64 float64 float64 float64 str64 str32 ... str64 float64 float64 object float64 float64 float64 bool bool float64 + ---------------- ---------- --------------- ----------- ------------------- ------------------- ------------------- ------------------- --------------- ----------- ... --------------------------- -------------------------------- -------------------------------- --------------------------------------- ------------------ ----------------- ------------- -------- --------- ------------------ + 2 JWST image NRCB2 F090W 0.795 1.005 0.2099999999999999 NIRCAM/IMAGE calibration ... P330-E 247.89368329393696 30.14664764798702 NIRCam Engineering Imaging 60471.51501850694 60471.5150444213 1.672 False True 0.8999999999999999 + -1 JWST image F770W 6.6000000000000005 8.8 2.2 MIRI/IMAGE science ... GC_132 266.9847738407929 -28.2785134188332 NIRCam Imaging -- -- 16.65 -- False 7.700000000000001 + 2 JWST image NRCB1 F115W 1.0130000000000001 1.282 0.2689999999999999 NIRCAM/IMAGE science ... OPH-CORE-Tile-2 246.5742328427403 -24.400402708813985 NIRCam Imaging 60040.4505228125 60040.45213829861 139.578 False True 1.1475 + -1 JWST image F200W 1.5999999999999999 2.3 0.7000000000000001 NIRISS/IMAGE science ... ECLIPTIC-RA80 124.019860712904 19.138870607305506 NIRCam Imaging -- -- 687.153 -- False 1.95 + 2 JWST image NRCA4 F090W 0.795 1.005 0.2099999999999999 NIRCAM/IMAGE science ... NGC-3643 170.3375891871629 2.985130126450244 NIRCam Imaging 61160.99709872685 61161.02642601852 2512.404 False True 0.8999999999999999 + 1 JWST image NRCB2 F200W 1.755 2.226 0.47100000000000003 NIRCAM/IMAGE science ... NGC-2835 139.46068692920358 -22.34831151818625 NIRCam Wide Field Slitless Spectroscopy 60801.00090825232 60801.0021509375 107.368 False False 1.9905 + 1 JWST image NRS1 F290LP;G395M 2.87 5.1000000000000005 2.2300000000000004 NIRSPEC/MSA science ... CEERS-NIRSPEC-P10-MR-MSATA 214.91812297644287 52.84822109140906 NIRSpec MultiObject Spectroscopy 59935.26957332176 59935.281561875 1021.222 False False 3.985 + ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... + 2 JWST image NRCALONG F277W 2.416 3.127 0.7110000000000001 NIRCAM/IMAGE science ... UNKNOWN 53.15896343558407 -27.887946072187304 NIRCam Imaging 60188.20147115741 60188.26546927083 5497.226 False True 2.7714999999999996 + 2 JWST image NRCALONG F250M 2.412 2.595 0.1830000000000002 NIRCAM/IMAGE science ... ECLIPTIC-RA60 106.97715213200891 27.54971496885016 NIRCam Imaging 59671.56161841319 59671.56534645833 311.366 False True 2.5035 + 3 JWST spectrum MULTIPLE F100LP;G140M 0.7 5.0 4.300000000000001 NIRSPEC/MSA science ... SUSPENSE_v10_correct_coords 150.53652421222472 2.4620705377064667 NIRSpec MultiObject Spectroscopy 60311.575211550924 60311.60931983796 2917.778 False True 2.85 + 2 JWST cube MIRIFUSHORT CH12-MEDIUM 5.66 10.129999999999999 4.47 MIRI/IFU science ... J0937+5628 144.408518766394 56.47750019928672 MIRI Medium Resolution Spectroscopy 60729.469680381946 60729.4761362037 557.783 True True 7.895 + 2 JWST image NRCA1 F200W 1.755 2.226 0.47100000000000003 NIRCAM/IMAGE science ... NGC-2835 139.45947726034913 -22.362472861173785 NIRCam Wide Field Slitless Spectroscopy 60800.84965297454 60800.85934589121 837.468 False True 1.9905 + 1 JWST image NRCB1 F140M 1.331 1.4789999999999999 0.1479999999999999 NIRCAM/IMAGE science ... UNKNOWN 3.6620150625204437 -30.5310785031449 NIRCam Imaging 60629.97793209491 60629.98899196759 934.099 False False 1.405 + 1 JWST image MIRIMAGE F1000W 9.0 11.0 1.9999999999999996 MIRI/IMAGE science ... NGC-3351 160.99495827680343 11.686557085203447 MIRI Imaging 60082.70275739583 60082.70311069444 30.525 False False 10.0 Query saving results in a file: .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> job = Jwst.launch_job("select top 100 * from jwst.main", dump_to_file=True) - >>> print(job) # doctest: +IGNORE_OUTPUT - Jobid: None - Phase: COMPLETED - Owner: None - Output file: 1635853688471D-result.vot.gz - Results: None - >>> r = job.get_results() - >>> r['instrument_name'] # doctest: +IGNORE_OUTPUT - - NIRCAM - NIRCAM - NIRISS/MSA - MIRI/IFU - NIRCAM/IMAGE - MIRI - ... + >>> result = Jwst.launch_job("select top 100 * from jwst.archive", output_file="archive.vot") 1.9 Asynchronous job removal @@ -593,7 +515,8 @@ To remove asynchronous .. doctest-skip:: >>> from astroquery.esa.jwst import Jwst - >>> job = Jwst.remove_jobs(["job_id_1", "job_id_2", ...]) + >>> job = Jwst.get_job_list()[0] + >>> job.delete() ----------------------- @@ -604,7 +527,7 @@ Authenticated users are able to access to TAP+ capabilities (shared tables, pers In order to authenticate a user, ``login`` method must be called. After a successful authentication, the user will be authenticated until ``logout`` method is called. -All previous methods (``query_object``, ``cone_search``, ``load_table``, ``load_tables``, ``launch_job``) explained for +All previous methods (``query_object``, ``cone_search``, ``get_table``, ``get_tables``, ``launch_job``) explained for non authenticated users are applicable for authenticated ones. The main differences are: @@ -619,48 +542,43 @@ The main differences are: Using the command line: -.. Skipping authentication requiring examples -.. doctest-skip:: +.. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login(user='userName', password='userPassword') + >>> Jwst.login(user='userName', password='userPassword') # doctest: +SKIP It is possible to use a file where the credentials are stored: *The file must containing user and password in two different lines.* -.. Skipping authentication requiring examples -.. doctest-skip:: +.. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login(credentials_file='my_credentials_file') + >>> Jwst.login(credentials_file='my_credentials_file') # doctest: +SKIP MAST tokens can also be used in command line functions: -.. Skipping authentication requiring examples -.. doctest-skip:: +.. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login(user='userName', password='userPassword', token='mastToken') + >>> Jwst.login(user='userName', password='userPassword', token='mastToken') # doctest: +SKIP If the user is logged in and a MAST token has not been included or must be changed, it can be specified using the ``set_token`` function. -.. Skipping authentication requiring examples -.. doctest-skip:: +.. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login(user='userName', password='userPassword') - >>> Jwst.set_token(token='mastToken') + >>> Jwst.login(user='userName', password='userPassword') # doctest: +SKIP + >>> Jwst.set_token(token='mastToken') # doctest: +SKIP To perform a logout: -.. Skipping authentication requiring examples -.. doctest-skip:: +.. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> Jwst.logout() + >>> Jwst.logout() # doctest: +SKIP @@ -670,29 +588,9 @@ To perform a logout: .. doctest-remote-data:: >>> from astroquery.esa.jwst import Jwst - >>> tables = Jwst.load_tables(only_names=True, include_shared_tables=True) - INFO: Retrieving tables... [astroquery.utils.tap.core] - INFO: Parsing tables... [astroquery.utils.tap.core] - INFO: Done. [astroquery.utils.tap.core] - >>> print(*(table.name for table in tables), sep="\n") # doctest: +IGNORE_OUTPUT - public.dual - tap_schema.columns - tap_schema.key_columns - tap_schema.keys - tap_schema.schemas - tap_schema.tables - jwst.artifact - jwst.chunk - jwst.main - jwst.observation - jwst.observationmember - jwst.part - jwst.plane - jwst.plane_inputs - ... - user_schema_1.table1 - user_schema_2.table1 - ... + >>> tables = Jwst.get_tables(only_names=True) + >>> tables + ['ivoa.ObsCore', 'jwst.archive', 'jwst.artifact', 'jwst.chunk', 'jwst.harvestskipuri', 'jwst.main', 'jwst.moc', 'jwst.mv_archive_angular', 'jwst.observation', 'jwst.observationmember', 'jwst.part', 'jwst.plane', 'jwst.plane_vis', 'public.dual', 'tap_config.coord_sys', 'tap_config.properties', 'tap_schema.columns', 'tap_schema.key_columns', 'tap_schema.keys', 'tap_schema.schemas', 'tap_schema.tables'] Reference/API