From d3ecf0ddd15b1b38a5497d4a9ee96771efe97d6f Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Tue, 5 Nov 2024 13:55:44 -0500 Subject: [PATCH 1/5] Parameter checking on Missions queries Mock MastMissions.get_column_list Increase code coverage --- CHANGES.rst | 2 + astroquery/mast/missions.py | 78 ++- .../mast/tests/data/mission_columns.json | 572 ++++++++++++++++++ astroquery/mast/tests/test_mast.py | 74 +-- astroquery/mast/tests/test_mast_remote.py | 18 + astroquery/mast/utils.py | 2 +- 6 files changed, 677 insertions(+), 69 deletions(-) create mode 100644 astroquery/mast/tests/data/mission_columns.json diff --git a/CHANGES.rst b/CHANGES.rst index e4eeb251d4..cc2d48a47d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -223,6 +223,8 @@ mast - Deprecated ``enable_cloud_dataset`` and ``disable_cloud_dataset`` in classes where they are non-operational. They will be removed in a future release. [#3113] +- Present users with an error when nonexistent query criteria are used in ``mast.MastMissions`` query functions. [#3126] + mpc ^^^ diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index 18df28ae4d..a467e342a1 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -6,7 +6,7 @@ This module contains methods for searching MAST missions. """ -import requests +import difflib import warnings from astropy.table import Table @@ -40,6 +40,7 @@ def __init__(self, *, mission='hst', service='search'): self.service = service self.mission = mission self.limit = 5000 + self.columns = dict() # Info about columns for each mission service_dict = {self.service: {'path': self.service, 'args': {}}} self._service_api_connection.set_service_params(service_dict, f"{self.service}/{self.mission}") @@ -69,6 +70,37 @@ def _parse_result(self, response, *, verbose=False): # Used by the async_to_syn return results + def _validate_criteria(self, **criteria): + """ + Check that criteria keyword arguments are valid column names for the mission. + Raises InvalidQueryError if a criteria argument is invalid. + + Parameters + ---------- + **kwargs + Keyword arguments representing criteria filters to apply. + + Raises + ------- + InvalidQueryError + If a keyword does not match any valid column names, an error is raised that suggests the closest + matching column name, if available. + """ + # Ensure that self.columns in populated + self.get_column_list() + + # Check each criteria argument for validity + valid_cols = self.columns[self.mission]['name'] + for kwd in criteria.keys(): + if kwd not in valid_cols and kwd not in self._search_option_fields: + closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) + error_msg = ( + f"Filter '{kwd}' does not exist. Did you mean '{closest_match[0]}'?" + if closest_match + else f"Filter '{kwd}' does not exist." + ) + raise InvalidQueryError(error_msg) + @class_or_instance def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs): """ @@ -104,6 +136,9 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs self.limit = limit + # Check that criteria arguments are valid + self._validate_criteria(**kwargs) + # Put coordinates and radius into consistent format coordinates = commons.parse_coordinates(coordinates) @@ -170,6 +205,9 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u. self.limit = limit + # Check that criteria arguments are valid + self._validate_criteria(**criteria) + if objectname or coordinates: coordinates = utils.parse_input_location(coordinates, objectname) @@ -237,25 +275,29 @@ def get_column_list(self): Returns ------- - response : `~astropy.table.Table` that contains columns names, types and their descriptions + response : `~astropy.table.Table` that contains columns names, types, and descriptions """ - url = f"{conf.server}/search/util/api/v0.1/column_list?mission={self.mission}" - - try: - results = requests.get(url) - results = results.json() - rows = [] - for result in results: - result.pop('field_name') - result.pop('queryable') - result.pop('indexed') - result.pop('default_output') - rows.append((result['column_name'], result['qual_type'], result['description'])) - data_table = Table(rows=rows, names=('name', 'data_type', 'description')) - return data_table - except Exception: - raise Exception(f"Error occurred while trying to get column list for mission {self.mission}") + if not self.columns.get(self.mission): + try: + # Send server request to get column list for current mission + params = {'mission': self.mission} + resp = utils._simple_request(f'{conf.server}/search/util/api/v0.1/column_list', params) + + # Parse JSON and extract necessary info + results = resp.json() + rows = [ + (result['column_name'], result['qual_type'], result['description']) + for result in results + ] + + # Create Table with parsed data + col_table = Table(rows=rows, names=('name', 'data_type', 'description')) + self.columns[self.mission] = col_table + except Exception: + raise Exception(f'Error occurred while trying to get column list for mission {self.mission}') + + return self.columns[self.mission] MastMissions = MastMissionsClass() diff --git a/astroquery/mast/tests/data/mission_columns.json b/astroquery/mast/tests/data/mission_columns.json new file mode 100644 index 0000000000..49ab7467a0 --- /dev/null +++ b/astroquery/mast/tests/data/mission_columns.json @@ -0,0 +1,572 @@ +[ + { + "field_name": "Search Position", + "description": "Search Position (RA and Dec) ", + "column_name": "search_pos", + "queryable": "no", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Dataset", + "description": "Data set name, the first character indicates instrument; L=COS; I=WFC3; J=ACS; N=NICMOS; O=STIS; U=WFPC2; W=WFPC; X=FOC; Y=FOS; Z=GHRS; F=FGS; V=HSP; nine-character name (e.g. J8BA7JCAQ, O4140Q020) ", + "column_name": "sci_data_set_name", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "yes", + "datatype": "char" + }, + { + "field_name": "Target Name", + "description": "Target name designated by the observer for the HST proposal; Uppercase; No blank characters; Spaces sometimes filled with - ; (e.g. A901-FIELD-25, NGC4486-POS1, 0537-441INCA221-36, ALPHA-CEN) ", + "column_name": "sci_targname", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "HAP", + "description": "Reports if there are any Hubble Advanced Products (HAP), enter 0 for no, 1 for yes ", + "column_name": "sci_hapnum", + "queryable": "yes", + "qual_type": "boolean", + "default_output": "yes", + "indexed": "no", + "datatype": "long" + }, + { + "field_name": "HASP", + "description": "Reports if there are any Hubble Advanced Spectral Products (HASP), enter 0 for no, 1 for yes ", + "column_name": "sci_haspnum", + "queryable": "yes", + "qual_type": "boolean", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Instrument", + "description": "Instrument used (e.g. ACS, COS, FGS, FOC, FOS, HRS, HSP, NICMOS, STIS, WFC3, WFPC, WFPC2) ", + "column_name": "sci_instrume", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Apertures", + "description": "Aperture configuration; WFPC2 (e.g. PC1, WF3, WFALL); ACS (e.g. WFC, HRC, SBC); STIS (e.g. 25MAMA, F25QTZ) ", + "column_name": "sci_aper_1234", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Filters/Gratings", + "description": "The filter(s) or grating(s) used (e.g. G160L, G270M, G230LB, F300W) ", + "column_name": "sci_spec_1234", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Exp Time", + "description": "Exposure time in seconds ", + "column_name": "sci_actual_duration", + "queryable": "yes", + "qual_type": "float", + "default_output": "yes", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Start Time", + "description": "Observation start time; The earliest in-flight data is available from Apr 24 1990 ", + "column_name": "sci_start_time", + "queryable": "yes", + "qual_type": "datetime", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Proposal ID", + "description": "ID number (integer) assigned to the proposal (e.g. 1012, 12045) ", + "column_name": "sci_pep_id", + "queryable": "yes", + "qual_type": "integer", + "default_output": "yes", + "indexed": "yes", + "datatype": "long" + }, + { + "field_name": "PI Last Name", + "description": "Principal Investigator (PI) last name; uppercase (e.g. FORD, LEMMON) ", + "column_name": "sci_pi_last_name", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "RA (J2000)", + "description": "Right Ascension (J2000), from 0 to 360, in degrees ", + "column_name": "sci_ra", + "queryable": "yes", + "qual_type": "ra", + "default_output": "yes", + "indexed": "yes", + "datatype": "double" + }, + { + "field_name": "Dec (J2000)", + "description": "Declination (J2000), from -90 to +90, in degrees ", + "column_name": "sci_dec", + "queryable": "yes", + "qual_type": "dec", + "default_output": "yes", + "indexed": "no", + "datatype": "double" + }, + { + "field_name": "Archive Class", + "description": "Archive ingest classification, where the only choices are CAL='calibrated science' and AST='astrometric' (FGS only) ", + "column_name": "sci_archive_class", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Prog ID", + "description": "Internal program ID, which is not the same as proposal ID; Three-character name (e.g. 8G6, 9FA, 8M8, 4QA) ", + "column_name": "sci_program_id", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "yes", + "datatype": "char" + }, + { + "field_name": "Obset ID", + "description": "Observation set ID, indicating the fourth and fifth letters of a dataset name. In most cases, this is usually the observation visit number; Two-character name (e.g. 10, 42, G1, GA, H6) ", + "column_name": "sci_obset_id", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "yes", + "datatype": "char" + }, + { + "field_name": "Obsnum", + "description": "Observation number, used in the Science Operations Ground System (SOGS) header; (e.g. 01, 02, RO, 011, AK, 070) ", + "column_name": "sci_obsnum", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "yes", + "datatype": "char" + }, + { + "field_name": "Asn ID", + "description": "The association ID for the dataset (result of combining individual exposures), identical to the data set name; If used to build an association, this field indicates the combined data sets (e.g. null, O4QA01010, J8ZO02010) ", + "column_name": "sci_asn_id", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Target Descrip", + "description": "Target description specified by the observer from a key field used in the Astronomer's Proposal Tool (APT); Uppercase (e.g. GALAXY;SEYFERT;STARBURST, ISM;HI CLOUD, STAR;PULSAR, PLANET;EXOSPHERE OF MARS) ", + "column_name": "sci_target_descrip", + "queryable": "yes", + "qual_type": "substring", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Broad Category", + "description": "One or more target classifications assigned by the observer from HST broad target categories; Uppercase (e.g. CLUSTER OF GALAXIES;GALAXY, EXT-CLUSTER;ISM, STAR;EXT-CLUSTER, SOLAR SYSTEM, CALIBRATION) ", + "column_name": "sci_broad_category", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Ref", + "description": "The number of known literature references associated with the proposal ID or data set name ", + "column_name": "sci_refnum", + "queryable": "yes", + "qual_type": "integer", + "default_output": "yes", + "indexed": "no", + "datatype": "long" + }, + { + "field_name": "Ra V1 (J2000)", + "description": "Right Ascension (J2000) at V1 position, from 0 to 360, in degrees, where the +V1 is pointing from the primary mirror along the optical axis toward the secondary mirror ", + "column_name": "sci_ra_v1", + "queryable": "yes", + "qual_type": "ra", + "default_output": "no", + "indexed": "no", + "datatype": "double" + }, + { + "field_name": "Dec V1 (J2000)", + "description": "Declination (J2000) at V1 position, from -90 to +90, in degrees, where the +V1 is pointing from the primary mirror along the optical axis toward the secondary mirror ", + "column_name": "sci_dec_v1", + "queryable": "yes", + "qual_type": "dec", + "default_output": "no", + "indexed": "no", + "datatype": "double" + }, + { + "field_name": "Exp Flag", + "description": "Exposure interruption indicator (e.g. NORMAL, INCOMPLETE, UNCERTAIN, INTERRUPTED, EXTENDED, PREDICTED, INDETERMINATE, UNKNOWN, EXCESSIVE DOW, DETOFF) ", + "column_name": "sci_expflag", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Instrument Config", + "description": "Instrument configuration (e.g. ACS/HRC, COS/FUV, STIS/CCD, STIS/FUV-MAMA, WFC3/IR, WFC3/UVIS, WFPC2) ", + "column_name": "sci_instrument_config", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Operating Mode", + "description": "Operating (data taking) mode (e.g. ACQ, RAMP, SCAN, SPEC, ACCUM, IMAGE, RAPID, UNKNOWN) ", + "column_name": "sci_operating_mode", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Spectral Res", + "description": "Spectral resolving power (R=\u03bb/\u0394\u03bb); Typical range is up to 114000 and depends on the instrument used ", + "column_name": "sci_spectral_res", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Spectrum End", + "description": "Maximum wavelength for spectrum, in Angstroms ", + "column_name": "sci_spectrum_end", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Spectrum Start", + "description": "Minimum wavelength for spectrum, in Angstroms ", + "column_name": "sci_spectrum_start", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Bandwidth", + "description": "Bandwidth of the data or root mean squares (RMS) bandwidth of the photmode (NICMOS) or filter plus detector (ACS) (e.g. WFC3 (2000-17000 A), COS/FUV (1150 - 2050 A)) ", + "column_name": "sci_bandwidth", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Central Wavelength", + "description": "Central wavelength, in Angstroms, of the bandpass for filters or gratings used; It depends on the instrument ", + "column_name": "sci_central_wavelength", + "queryable": "yes", + "qual_type": "float", + "default_output": "yes", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Dispersion", + "description": "Dispersion of grating; Typical range is up to 70.28 Angstroms/pixel and depends on the instrument used ", + "column_name": "sci_dispersion", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Pixel Res", + "description": "Pixel resolution (arcsec/pixel) ", + "column_name": "sci_pixel_res", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "FGS Lock", + "description": "Fine Guidance Sensors (FGS) lock status (e.g. FINE, GYROS, COARSE, FINE/GYRO, UNKNOWN) ", + "column_name": "sci_fgslock", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "MT Flag", + "description": "Moving Target Flag (T=True means a Moving Target, or F=False) ", + "column_name": "sci_mtflag", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Release Date", + "description": "Date of public release; The earliest date is available from May 9, 1990 ", + "column_name": "sci_release_date", + "queryable": "yes", + "qual_type": "datetime", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Stop Time", + "description": "Observation stop time; The earliest in-flight data is available from Apr 24 1990 ", + "column_name": "sci_stop_time", + "queryable": "yes", + "qual_type": "datetime", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "V3 Pos Angle", + "description": "The V3 position angle, from 0 to 360, in degrees; It is measured from North to East toward the projection of the V3 axis on the sky; V3 is an axis of the HST coordinate system aligning with the High Gain Antenna ", + "column_name": "sci_v3_pos_angle", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Ecliptic Latitude", + "description": "Ecliptic Latitude (J2000), from -90 to +90, in degrees ", + "column_name": "sci_ecliptic_latitude", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Ecliptic Longitude", + "description": "Ecliptic Longitude (J2000), from 0 to 360, in degrees ", + "column_name": "sci_ecliptic_longitude", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Galactic Latitude", + "description": "Galactic Latitude (J2000), from -90 to +90, in degrees ", + "column_name": "sci_galactic_latitude", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Galactic Longitude", + "description": "Galactic Longitude (J2000), from 0 to 360, in degrees ", + "column_name": "sci_galactic_longitude", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Sun Alt", + "description": "Altitude of the Sun above the Earth's limb from -66 to 114, in degrees ", + "column_name": "sci_sun_alt", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "PA Aper", + "description": "Position angle of aperture, from -180 to +360, in degrees, which is the angle measured from North to East toward the aperture's y-axis ", + "column_name": "sci_pa_aper", + "queryable": "yes", + "qual_type": "float", + "default_output": "no", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "COSTAR", + "description": "This indicates if the Corrective Optics Space Telescope Axial Replacement (COSTAR) was deployed; (T=True means deployed, or F=False) ", + "column_name": "sci_costar_deploy", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "AEC", + "description": "Archived Exposures Catalog (AEC); Purpose of observation (C=Calibration, S=Science) ", + "column_name": "sci_aec", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "FOV Config", + "description": "Field of view (FOV) configuration (LAUNCH, SM-1, SM-2, SM-3, SM-4) ", + "column_name": "sci_fov_config", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Generation Date", + "description": "The date when associated data products were generated by the processing pipeline; The earliest data is available from Feb 12 1992 ", + "column_name": "sci_generation_date", + "queryable": "yes", + "qual_type": "datetime", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Archive Date", + "description": "The date when archiving was completed; The earliest data is available from Dec 23, 1993 ", + "column_name": "sci_archive_date", + "queryable": "yes", + "qual_type": "datetime", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Preview Name", + "description": "Preview name, generally the same as the data set name (e.g. J9FAHGH5Q) ", + "column_name": "sci_preview_name", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Obs Type", + "description": "Observation category; ASTROMETERY (FGS only), IMAGE, PHOTOMETRY (HSP only), SPECTRUM ", + "column_name": "sci_obs_type", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Scan Type", + "description": "Observation scan mode; one of two types of spatial scan (C=observation taken in continuous scan mode, or D=observation taken in scan mode with a dwell) ", + "column_name": "scp_scan_type", + "queryable": "yes", + "qual_type": "string", + "default_output": "yes", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "High-Level Science Products", + "description": "Number of known High Level Science Products ", + "column_name": "sci_hlsp", + "queryable": "yes", + "qual_type": "integer", + "default_output": "yes", + "indexed": "no", + "datatype": "long" + }, + { + "field_name": "Ang Sep (')", + "description": "Angular Separation ", + "column_name": "ang_sep", + "queryable": "yes", + "qual_type": "float", + "default_output": "yes", + "indexed": "no", + "datatype": "float" + }, + { + "field_name": "Status", + "description": "Data access status (PUBLIC or PROPRIETARY=exclusive data) ", + "column_name": "sci_status", + "queryable": "yes", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + }, + { + "field_name": "Search Key", + "description": "Search Key (Concat of sci_data_set_name and search_pos) ", + "column_name": "search_key", + "queryable": "no", + "qual_type": "string", + "default_output": "no", + "indexed": "no", + "datatype": "char" + } +] \ No newline at end of file diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index d1fefec201..3b47b23e92 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -22,6 +22,7 @@ DATA_FILES = {'Mast.Caom.Cone': 'caom.json', 'Mast.Name.Lookup': 'resolver.json', 'mission_search_results': 'mission_results.json', + 'mission_columns': 'mission_columns.json', 'columnsconfig': 'columnsconfig.json', 'ticcolumns': 'ticcolumns.json', 'ticcol_filtered': 'ticcolumns_filtered.json', @@ -62,7 +63,7 @@ def data_path(filename): def patch_post(request): mp = request.getfixturevalue("monkeypatch") - mp.setattr(mast.utils, '_simple_request', resolver_mockreturn) + mp.setattr(mast.utils, '_simple_request', request_mockreturn) mp.setattr(mast.discovery_portal.PortalAPI, '_request', post_mockreturn) mp.setattr(mast.services.ServiceAPI, '_request', service_mockreturn) mp.setattr(mast.auth.MastAuth, 'session_info', session_info_mockreturn) @@ -128,8 +129,11 @@ def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, use return MockResponse(content) -def resolver_mockreturn(*args, **kwargs): - filename = data_path(DATA_FILES["Mast.Name.Lookup"]) +def request_mockreturn(url, params=None): + if 'column_list' in url: + filename = data_path(DATA_FILES['mission_columns']) + elif 'Mast.Name.Lookup' in params: + filename = data_path(DATA_FILES["Mast.Name.Lookup"]) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -210,58 +214,28 @@ def test_missions_query_region(patch_post): def test_missions_query_criteria_async(patch_post): - pep_id = {'sci_pep_id': '12556'} - obs_type = {'sci_obs_type': "SPECTRUM"} - instruments = {'sci_instrume': "stis,acs,wfc3,cos,fos,foc,nicmos,ghrs"} - datasets = {'sci_data_set_name': ""} - pi_lname = {'sci_pi_last_name': ""} - spec_1234 = {'sci_spec_1234': ""} - release_date = {'sci_release_date': ""} - start_time = {'sci_start_time': ""} - obs_type = {'sci_obs_type': 'all'} - aec = {'sci_aec': 'S'} - responses = mast.MastMissions.query_criteria_async(coordinates=regionCoords, - radius=3, - conditions=[pep_id, - obs_type, - instruments, - datasets, - pi_lname, - spec_1234, - release_date, - start_time, - obs_type, - aec]) + responses = mast.MastMissions.query_criteria_async( + coordinates=regionCoords, + radius=3, + sci_pep_id=12556, + sci_obs_type='SPECTRUM', + sci_instrume='stis,acs,wfc3,cos,fos,foc,nicmos,ghrs', + sci_aec='S' + ) assert isinstance(responses, MockResponse) def test_missions_query_criteria_async_with_missing_results(patch_post): - pep_id = {'sci_pep_id': '12556'} - obs_type = {'sci_obs_type': "SPECTRUM"} - instruments = {'sci_instrume': "stis,acs,wfc3,cos,fos,foc,nicmos,ghrs"} - datasets = {'sci_data_set_name': ""} - pi_lname = {'sci_pi_last_name': ""} - spec_1234 = {'sci_spec_1234': ""} - release_date = {'sci_release_date': ""} - start_time = {'sci_start_time': ""} - obs_type = {'sci_obs_type': 'all'} - aec = {'sci_aec': 'S'} - aperture = {'sci_aper_1234': 'WF3'} - with pytest.raises(KeyError): - responses = mast.MastMissions.query_criteria_async(coordinates=regionCoords, - radius=5, - conditions=[pep_id, - obs_type, - instruments, - datasets, - pi_lname, - spec_1234, - release_date, - start_time, - obs_type, - aec, - aperture]) + responses = mast.MastMissions.query_criteria_async( + coordinates=regionCoords, + radius=5, + sci_pep_id=12556, + sci_obs_type='SPECTRUM', + sci_instrume='stis,acs,wfc3,cos,fos,foc,nicmos,ghrs', + sci_aec='S', + sci_aper_1234='WF3' + ) _json_to_table(json.loads(responses), 'results') diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 911d778628..001bfc2fe6 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -136,6 +136,24 @@ def test_missions_query_criteria(self): MastMissions.query_criteria(coordinates="245.89675 -26.52575", radius=1) + def test_missions_query_criteria_invalid_keyword(self): + # Attempt to make a criteria query with invalid keyword + with pytest.raises(InvalidQueryError) as err_no_alt: + MastMissions.query_criteria(select_cols=['sci_targname'], + not_a_keyword='test') + assert "Filter 'not_a_keyword' does not exist." in str(err_no_alt.value) + + # Attempt to make a region query with invalid keyword + with pytest.raises(InvalidQueryError) as err_no_alt: + MastMissions.query_region(coordinates="245.89675 -26.52575", + invalid_keyword='test') + assert "Filter 'invalid_keyword' does not exist." in str(err_no_alt.value) + + # Keyword is close enough for difflib to offer alternative + with pytest.raises(InvalidQueryError) as err_with_alt: + MastMissions.query_criteria(search_position='30 30') + assert 'search_pos' in str(err_with_alt.value) + ################### # MastClass tests # ################### diff --git a/astroquery/mast/utils.py b/astroquery/mast/utils.py index c8bb384ffe..9f7a25d731 100644 --- a/astroquery/mast/utils.py +++ b/astroquery/mast/utils.py @@ -70,7 +70,7 @@ def parse_type(dbtype): }.get(dbtype, (dbtype, dbtype, dbtype)) -def _simple_request(url, params): +def _simple_request(url, params=None): """ Light wrapper on requests.session().get basically to make monkey patched testing easier/more effective. """ From 9d659e7d7c4723d1566ac87e465b963a10dfcf2a Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 6 Nov 2024 09:08:38 -0500 Subject: [PATCH 2/5] Rename **kwargs, info to regenerate column file, parameter checking on collections --- astroquery/mast/collections.py | 58 +++++++++++++++++---------- astroquery/mast/missions.py | 36 ++++++++--------- astroquery/mast/tests/data/README.rst | 17 ++++++++ 3 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 astroquery/mast/tests/data/README.rst diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 9d56e31ce1..97d56f078a 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -6,6 +6,7 @@ This module contains various methods for querying MAST collections such as catalogs. """ +import difflib import warnings import os import time @@ -60,7 +61,7 @@ def _parse_result(self, response, *, verbose=False): @class_or_instance def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", - version=None, pagesize=None, page=None, **kwargs): + version=None, pagesize=None, page=None, **criteria): """ Given a sky position and radius, returns a list of catalog entries. See column documentation for specific catalogs `here `__. @@ -88,7 +89,7 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", Default None. Can be used to override the default behavior of all results being returned to obtain a specific page of results. - **kwargs + **criteria Other catalog-specific keyword args. These can be found in the (service documentation)[https://mast.stsci.edu/api/v0/_services.html] for specific catalogs. For example one can specify the magtype for an HSC search. @@ -109,10 +110,18 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", 'dec': coordinates.dec.deg, 'radius': radius.deg} + # valid criteria keywords + valid_criteria = [] + # Determine API connection and service name if catalog.lower() in self._service_api_connection.SERVICES: self._current_connection = self._service_api_connection service = catalog + + # adding additional user specified parameters + for prop, value in criteria.items(): + params[prop] = value + else: self._current_connection = self._portal_api_connection @@ -125,19 +134,20 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning) service = "Mast.Hsc.Db.v3" - self.catalog_limit = kwargs.get('nr', 50000) - # Hsc specific parameters (can be overridden by user) - params['nr'] = 50000 - params['ni'] = 1 - params['magtype'] = 1 + self.catalog_limit = criteria.pop('nr', 50000) + valid_criteria = ['nr', 'ni', 'magtype'] + params['nr'] = self.catalog_limit + params['ni'] = criteria.pop('ni', 1) + params['magtype'] = criteria.pop('magtype', 1) elif catalog.lower() == "galex": service = "Mast.Galex.Catalog" - self.catalog_limit = kwargs.get('maxrecords', 50000) + self.catalog_limit = criteria.get('maxrecords', 50000) # galex specific parameters (can be overridden by user) - params['maxrecords'] = 50000 + valid_criteria = ['maxrecords'] + params['maxrecords'] = criteria.pop('maxrecords', 50000) elif catalog.lower() == "gaia": if version == 1: @@ -158,9 +168,16 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", service = "Mast.Catalogs." + catalog + ".Cone" self.catalog_limit = None - # adding additional user specified parameters - for prop, value in kwargs.items(): - params[prop] = value + # additional user-specified parameters are not valid + if criteria: + key = next(iter(criteria)) + closest_match = difflib.get_close_matches(key, valid_criteria, n=1) + error_msg = ( + f"Filter '{key}' does not exist for catalog {catalog}. Did you mean '{closest_match[0]}'?" + if closest_match + else f"Filter '{key}' does not exist for catalog {catalog}." + ) + raise InvalidQueryError(error_msg) # Parameters will be passed as JSON objects only when accessing the PANSTARRS API use_json = catalog.lower() == 'panstarrs' @@ -170,7 +187,7 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", @class_or_instance def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc", - pagesize=None, page=None, version=None, **kwargs): + pagesize=None, page=None, version=None, **criteria): """ Given an object name, returns a list of catalog entries. See column documentation for specific catalogs `here `__. @@ -197,7 +214,7 @@ def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc", to obtain a specific page of results. version : int, optional Version number for catalogs that have versions. Default is highest version. - **kwargs + **criteria Catalog-specific keyword args. These can be found in the `service documentation `__. for specific catalogs. For example one can specify the magtype for an HSC search. @@ -215,7 +232,7 @@ def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc", version=version, pagesize=pagesize, page=page, - **kwargs) + **criteria) @class_or_instance def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria): @@ -295,26 +312,25 @@ def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria) if coordinates or objectname: service += ".Position" service += ".Rows" # Using the rowstore version of the query for speed - filters = self._current_connection.build_filter_set("Mast.Catalogs.Tess.Cone", - service, **criteria) + column_config_name = "Mast.Catalogs.Tess.Cone" params["columns"] = "*" elif catalog.lower() == "ctl": service = "Mast.Catalogs.Filtered.Ctl" if coordinates or objectname: service += ".Position" service += ".Rows" # Using the rowstore version of the query for speed - filters = self._current_connection.build_filter_set("Mast.Catalogs.Tess.Cone", - service, **criteria) + column_config_name = "Mast.Catalogs.Tess.Cone" params["columns"] = "*" elif catalog.lower() == "diskdetective": service = "Mast.Catalogs.Filtered.DiskDetective" if coordinates or objectname: service += ".Position" - filters = self._current_connection.build_filter_set("Mast.Catalogs.Dd.Cone", - service, **criteria) + column_config_name = "Mast.Catalogs.Dd.Cone" else: raise InvalidQueryError("Criteria query not available for {}".format(catalog)) + filters = self._current_connection.build_filter_set(column_config_name, service, **criteria) + if not filters: raise InvalidQueryError("At least one non-positional criterion must be supplied.") params["filters"] = filters diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index a467e342a1..34e70a2c1c 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -77,7 +77,7 @@ def _validate_criteria(self, **criteria): Parameters ---------- - **kwargs + **criteria Keyword arguments representing criteria filters to apply. Raises @@ -102,7 +102,7 @@ def _validate_criteria(self, **criteria): raise InvalidQueryError(error_msg) @class_or_instance - def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs): + def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offset=0, **criteria): """ Given a sky position and radius, returns a list of matching dataset IDs. @@ -122,12 +122,11 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs offset : int Optional and default is 0 the number of records you wish to skip before selecting records. - **kwargs - Other mission-specific keyword args. - Any invalid keys are ignored by the API. - All valid key names can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` + **criteria + Other mission-specific criteria arguments. + All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. - For example one can specify the output columns(select_cols) or use other filters(conditions) + For example, one can specify the output columns(select_cols) or use other filters(conditions). Returns ------- @@ -137,7 +136,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs self.limit = limit # Check that criteria arguments are valid - self._validate_criteria(**kwargs) + self._validate_criteria(**criteria) # Put coordinates and radius into consistent format coordinates = commons.parse_coordinates(coordinates) @@ -154,7 +153,7 @@ def query_region_async(self, coordinates, *, radius=3*u.arcmin, limit=5000, offs params['conditions'] = [] # adding additional user specified parameters - for prop, value in kwargs.items(): + for prop, value in criteria.items(): if prop not in self._search_option_fields: params['conditions'].append({prop: value}) else: @@ -189,12 +188,11 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u. select_cols: list names of columns that will be included in the astropy table **criteria - Criteria to apply. At least one non-positional criteria must be supplied. + Criteria to apply. At least one non-positional criterion must be supplied. Valid criteria are coordinates, objectname, radius (as in `~astroquery.mast.missions.MastMissionsClass.query_region` and `~astroquery.mast.missions.MastMissionsClass.query_object` functions), and all fields listed in the column documentation for the mission being queried. - Any invalid keys passed in criteria are ignored by the API. List of all valid fields that can be used to match results on criteria can be retrieved by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. @@ -234,7 +232,7 @@ def query_criteria_async(self, *, coordinates=None, objectname=None, radius=3*u. return self._service_api_connection.service_request_async(self.service, params, use_json=True) @class_or_instance - def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **kwargs): + def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offset=0, **criteria): """ Given an object name, returns a list of matching rows. @@ -253,11 +251,11 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse offset : int Optional and default is 0. the number of records you wish to skip before selecting records. - **kwargs - Mission-specific keyword args. - Any invalid keys are ignored by the API. - All valid keys can be found by calling `~astroquery.mast.missions.MastMissionsClass.get_column_list` + **criteria + Other mission-specific criteria arguments. + All valid filters can be found using `~astroquery.mast.missions.MastMissionsClass.get_column_list` function. + For example, one can specify the output columns(select_cols) or use other filters(conditions). Returns ------- @@ -266,7 +264,7 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse coordinates = utils.resolve_object(objectname) - return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **kwargs) + return self.query_region_async(coordinates, radius=radius, limit=limit, offset=offset, **criteria) @class_or_instance def get_column_list(self): @@ -294,8 +292,8 @@ def get_column_list(self): # Create Table with parsed data col_table = Table(rows=rows, names=('name', 'data_type', 'description')) self.columns[self.mission] = col_table - except Exception: - raise Exception(f'Error occurred while trying to get column list for mission {self.mission}') + except Exception as ex: + raise RuntimeError(f'Error occurred while trying to get column list for mission {self.mission}: {ex}') return self.columns[self.mission] diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst new file mode 100644 index 0000000000..8a7427033a --- /dev/null +++ b/astroquery/mast/tests/data/README.rst @@ -0,0 +1,17 @@ +=============== +MAST Test Data +=============== + +This directory contains sample data that is used to mock functions in `~astroquery.mast.tests.test_mast.py`. + +To generate `~astroquery.mast.tests.data.mission_columns.json`, use the following: + +.. code-block:: python + + >>> import json + >>> from astroquery.mast import utils + + >>> params = {'mission': 'hst'} + >>> resp = utils._simple_request(f'https://mast.stsci.edu/search/util/api/v0.1/column_list', {'mission': 'hst'}) + >>> with open('mission_columns.json', 'w') as file: + >>> json.dump(resp.json(), file, indent=4) From 18772c5e0919daef81cd2a3ae6f489be74de4a5f Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 6 Nov 2024 09:44:15 -0500 Subject: [PATCH 3/5] Better exception handling, fix README code fix JSONDecodeError import --- astroquery/mast/missions.py | 14 +++++++++++++- astroquery/mast/tests/data/README.rst | 9 ++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index 34e70a2c1c..b29f8f466a 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -7,11 +7,13 @@ """ import difflib +from json import JSONDecodeError import warnings from astropy.table import Table import astropy.units as u import astropy.coordinates as coord +from requests import RequestException from astroquery.utils import commons, async_to_sync from astroquery.utils.class_or_instance import class_or_instance @@ -292,8 +294,18 @@ def get_column_list(self): # Create Table with parsed data col_table = Table(rows=rows, names=('name', 'data_type', 'description')) self.columns[self.mission] = col_table + except JSONDecodeError as ex: + raise JSONDecodeError(f'Failed to decode JSON response while attempting to get column list' + f' for mission {self.mission}: {ex}') + except RequestException as ex: + raise ConnectionError(f'Failed to connect to the server while attempting to get column list' + f' for mission {self.mission}: {ex}') + except KeyError as ex: + raise KeyError(f'Expected key not found in response data while attempting to get column list' + f' for mission {self.mission}: {ex}') except Exception as ex: - raise RuntimeError(f'Error occurred while trying to get column list for mission {self.mission}: {ex}') + raise RuntimeError(f'An unexpected error occurred while attempting to get column list' + f' for mission {self.mission}: {ex}') return self.columns[self.mission] diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 8a7427033a..9821652087 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -6,12 +6,11 @@ This directory contains sample data that is used to mock functions in `~astroque To generate `~astroquery.mast.tests.data.mission_columns.json`, use the following: -.. code-block:: python +.. doctest-remote-data:: >>> import json >>> from astroquery.mast import utils - - >>> params = {'mission': 'hst'} - >>> resp = utils._simple_request(f'https://mast.stsci.edu/search/util/api/v0.1/column_list', {'mission': 'hst'}) + ... + >>> resp = utils._simple_request('https://mast.stsci.edu/search/util/api/v0.1/column_list', {'mission': 'hst'}) >>> with open('mission_columns.json', 'w') as file: - >>> json.dump(resp.json(), file, indent=4) + ... json.dump(resp.json(), file, indent=4) From 30c69ee8f1977e161834035bf51c5dedadc0045b Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 6 Nov 2024 16:37:19 -0500 Subject: [PATCH 4/5] Validate panstarrs criteria --- CHANGES.rst | 3 + astroquery/mast/__init__.py | 3 + astroquery/mast/collections.py | 132 +- astroquery/mast/missions.py | 7 +- astroquery/mast/tests/data/README.rst | 11 + .../mast/tests/data/panstarrs_columns.json | 2358 +++++++++++++++++ astroquery/mast/tests/test_mast.py | 6 +- astroquery/mast/tests/test_mast_remote.py | 19 + 8 files changed, 2527 insertions(+), 12 deletions(-) create mode 100644 astroquery/mast/tests/data/panstarrs_columns.json diff --git a/CHANGES.rst b/CHANGES.rst index cc2d48a47d..897d7f82d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -225,6 +225,9 @@ mast - Present users with an error when nonexistent query criteria are used in ``mast.MastMissions`` query functions. [#3126] +- Present users with an error when nonexistent query criteria are used in ``mast.Catalogs.query_region`` and + ``mast.Catalogs.query_object`` [#3126] + mpc ^^^ diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index ffaae30737..96028a89e4 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -20,6 +20,9 @@ class Conf(_config.ConfigNamespace): ssoserver = _config.ConfigItem( 'https://ssoportal.stsci.edu', 'MAST SSO Portal server.') + catalogs_server = _config.ConfigItem( + 'https://catalogs.mast.stsci.edu', + 'Catalogs.MAST server.') timeout = _config.ConfigItem( 600, 'Time limit for requests from the STScI server.') diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 97d56f078a..3eea6eb806 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -7,11 +7,12 @@ """ import difflib +from json import JSONDecodeError import warnings import os import time -from requests import HTTPError +from requests import HTTPError, RequestException import astropy.units as u import astropy.coordinates as coord @@ -22,7 +23,7 @@ from ..utils.class_or_instance import class_or_instance from ..exceptions import InvalidQueryError, MaxResultsWarning, InputWarning -from . import utils +from . import utils, conf from .core import MastQueryWithLogin @@ -43,11 +44,13 @@ def __init__(self): services = {"panstarrs": {"path": "panstarrs/{data_release}/{table}.json", "args": {"data_release": "dr2", "table": "mean"}}} + self._catalogs_mast_search_options = ['columns', 'sort_by', 'table', 'data_release'] self._service_api_connection.set_service_params(services, "catalogs", True) self.catalog_limit = None self._current_connection = None + self._service_columns = dict() # Info about columns for Catalogs.MAST services def _parse_result(self, response, *, verbose=False): @@ -59,6 +62,99 @@ def _parse_result(self, response, *, verbose=False): return results_table + def _get_service_col_config(self, catalog, release='dr2', table='mean'): + """ + For a given Catalogs.MAST catalog, return a list of all searchable columns and their descriptions. + As of now, this function is exclusive to the Pan-STARRS catalog. + + Parameters + ---------- + catalog : str + The catalog to be queried. + release : str, optional + Catalog data release to query from. + table : str, optional + Catalog table to query from. + + Returns + ------- + response : `~astropy.table.Table` that contains columns names, types, and descriptions + """ + # Only supported for PanSTARRS currently + if catalog != 'panstarrs': + return + + service_key = (catalog, release, table) + if service_key not in self._service_columns: + try: + # Send server request to get column list for given parameters + request_url = f'{conf.catalogs_server}/api/v0.1/{catalog}/{release}/{table}/metadata.json' + resp = utils._simple_request(request_url) + + # Parse JSON and extract necessary info + results = resp.json() + rows = [ + (result['column_name'], result['db_type'], result['description']) + for result in results + ] + + # Create Table with parsed data + col_table = Table(rows=rows, names=('name', 'data_type', 'description')) + self._service_columns[service_key] = col_table + + except JSONDecodeError as ex: + raise JSONDecodeError(f'Failed to decode JSON response while attempting to get column list' + f' for {catalog} catalog {table}, {release}: {ex}') + except RequestException as ex: + raise ConnectionError(f'Failed to connect to the server while attempting to get column list' + f' for {catalog} catalog {table}, {release}: {ex}') + except KeyError as ex: + raise KeyError(f'Expected key not found in response data while attempting to get column list' + f' for {catalog} catalog {table}, {release}: {ex}') + except Exception as ex: + raise RuntimeError(f'An unexpected error occurred while attempting to get column list' + f' for {catalog} catalog {table}, {release}: {ex}') + + return self._service_columns[service_key] + + def _validate_service_criteria(self, catalog, **criteria): + """ + Check that criteria keyword arguments are valid column names for the service. + Raises InvalidQueryError if a criteria argument is invalid. + + Parameters + ---------- + catalog : str + The catalog to be queried. + **criteria + Keyword arguments representing criteria filters to apply. + + Raises + ------- + InvalidQueryError + If a keyword does not match any valid column names, an error is raised that suggests the closest + matching column name, if available. + """ + # Ensure that self._service_columns is populated + release = criteria.get('data_release', 'dr2') + table = criteria.get('table', 'mean') + col_config = self._get_service_col_config(catalog, release, table) + + if col_config: + # Check each criteria argument for validity + valid_cols = list(col_config['name']) + self._catalogs_mast_search_options + for kwd in criteria.keys(): + col = next((name for name in valid_cols if name.lower() == kwd.lower()), None) + if not col: + closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) + error_msg = ( + f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}. " + f"Did you mean '{closest_match[0]}'?" + if closest_match + else f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}." + ) + raise InvalidQueryError(error_msg) + @class_or_instance def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", version=None, pagesize=None, page=None, **criteria): @@ -92,7 +188,15 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", **criteria Other catalog-specific keyword args. These can be found in the (service documentation)[https://mast.stsci.edu/api/v0/_services.html] - for specific catalogs. For example one can specify the magtype for an HSC search. + for specific catalogs. For example, one can specify the magtype for an HSC search. + For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument + should be either an acceptable value for that parameter, or a list consisting values, or tuples of + decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, + consisting of a list of column names. Results may also be sorted through the query with the parameter + sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or + tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). + Detailed information of Catalogs.MAST criteria usage can + be found `here `__. Returns ------- @@ -110,14 +214,14 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", 'dec': coordinates.dec.deg, 'radius': radius.deg} - # valid criteria keywords - valid_criteria = [] - # Determine API connection and service name if catalog.lower() in self._service_api_connection.SERVICES: self._current_connection = self._service_api_connection service = catalog + # validate user criteria + self._validate_service_criteria(catalog.lower(), **criteria) + # adding additional user specified parameters for prop, value in criteria.items(): params[prop] = value @@ -125,6 +229,9 @@ def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", else: self._current_connection = self._portal_api_connection + # valid criteria keywords + valid_criteria = [] + # Sorting out the non-standard portal service names if catalog.lower() == "hsc": if version == 2: @@ -217,7 +324,15 @@ def query_object_async(self, objectname, *, radius=0.2*u.deg, catalog="Hsc", **criteria Catalog-specific keyword args. These can be found in the `service documentation `__. - for specific catalogs. For example one can specify the magtype for an HSC search. + for specific catalogs. For example, one can specify the magtype for an HSC search. + For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument + should be either an acceptable value for that parameter, or a list consisting values, or tuples of + decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, + consisting of a list of column names. Results may also be sorted through the query with the parameter + sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or + tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). + Detailed information of Catalogs.MAST criteria usage can + be found `here `__. Returns ------- @@ -298,6 +413,9 @@ def query_criteria_async(self, catalog, *, pagesize=None, page=None, **criteria) self._current_connection = self._service_api_connection service = catalog + # validate user criteria + self._validate_service_criteria(catalog.lower(), **criteria) + if not self._current_connection.check_catalogs_criteria_params(criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") diff --git a/astroquery/mast/missions.py b/astroquery/mast/missions.py index b29f8f466a..0d56fcda11 100644 --- a/astroquery/mast/missions.py +++ b/astroquery/mast/missions.py @@ -88,13 +88,14 @@ def _validate_criteria(self, **criteria): If a keyword does not match any valid column names, an error is raised that suggests the closest matching column name, if available. """ - # Ensure that self.columns in populated + # Ensure that self.columns is populated self.get_column_list() # Check each criteria argument for validity - valid_cols = self.columns[self.mission]['name'] + valid_cols = list(self.columns[self.mission]['name']) + self._search_option_fields for kwd in criteria.keys(): - if kwd not in valid_cols and kwd not in self._search_option_fields: + col = next((name for name in valid_cols if name.lower() == kwd.lower()), None) + if not col: closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) error_msg = ( f"Filter '{kwd}' does not exist. Did you mean '{closest_match[0]}'?" diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 9821652087..cfcb0dbff9 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -14,3 +14,14 @@ To generate `~astroquery.mast.tests.data.mission_columns.json`, use the followin >>> resp = utils._simple_request('https://mast.stsci.edu/search/util/api/v0.1/column_list', {'mission': 'hst'}) >>> with open('mission_columns.json', 'w') as file: ... json.dump(resp.json(), file, indent=4) + +To generate `~astroquery.mast.tests.data.panstarrs_columns.json`, use the following: + +.. doctest-remote-data:: + + >>> import json + >>> from astroquery.mast import utils + ... + >>> resp = utils._simple_request('https://catalogs.mast.stsci.edu/api/v0.1/panstarrs/dr2/mean/metadata.json') + >>> with open('panstarrs_columns.json', 'w') as file: + ... json.dump(resp.json(), file, indent=4) diff --git a/astroquery/mast/tests/data/panstarrs_columns.json b/astroquery/mast/tests/data/panstarrs_columns.json new file mode 100644 index 0000000000..bd1cd71da1 --- /dev/null +++ b/astroquery/mast/tests/data/panstarrs_columns.json @@ -0,0 +1,2358 @@ +[ + { + "name": "objName", + "db_type": "STRING", + "column_name": "objName", + "description": "IAU name for this object.", + "unit": "NULL", + "ucd": "meta.id;meta.main", + "utype": "caom:ps1.catalogEntity.id", + "datatype": "char", + "size": "32", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "1", + "arraysize": "32*", + "search_priority": "1", + "default_value": "NULL", + "type": "char" + }, + { + "name": "objAltName1", + "db_type": "STRING", + "column_name": "objAltName1", + "description": "Alternate name for this object.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "char", + "size": "32", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "2", + "arraysize": "32*", + "search_priority": "2", + "default_value": "NULL", + "type": "char" + }, + { + "name": "objAltName2", + "db_type": "STRING", + "column_name": "objAltName2", + "description": "Altername name for this object.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "char", + "size": "32", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "3", + "arraysize": "32*", + "search_priority": "3", + "default_value": "NULL", + "type": "char" + }, + { + "name": "objAltName3", + "db_type": "STRING", + "column_name": "objAltName3", + "description": "Altername name for this object.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "char", + "size": "32", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "4", + "arraysize": "32*", + "search_priority": "4", + "default_value": "NULL", + "type": "char" + }, + { + "name": "objID", + "db_type": "BIGINT", + "column_name": "objID", + "description": "Unique object identifier.", + "unit": "NULL", + "ucd": "meta.id;meta.main", + "utype": "NULL", + "datatype": "long", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "5", + "arraysize": "8", + "search_priority": "5", + "default_value": "NULL", + "type": "long" + }, + { + "name": "uniquePspsOBid", + "db_type": "BIGINT", + "column_name": "uniquePspsOBid", + "description": "Unique internal PSPS object identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "long", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "6", + "arraysize": "8", + "search_priority": "6", + "default_value": "NULL", + "type": "long" + }, + { + "name": "ippObjID", + "db_type": "BIGINT", + "column_name": "ippObjID", + "description": "IPP internal object identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "long", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "7", + "arraysize": "8", + "search_priority": "7", + "default_value": "NULL", + "type": "long" + }, + { + "name": "surveyID", + "db_type": "TINYINT", + "column_name": "surveyID", + "description": "Survey identifier. Details in the Survey table.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "unsignedByte", + "size": "1", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "8", + "arraysize": "1", + "search_priority": "8", + "default_value": "NULL", + "type": "unsignedByte" + }, + { + "name": "htmID", + "db_type": "BIGINT", + "column_name": "htmID", + "description": "Hierarchical triangular mesh (Szalay 2007) index.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "long", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "9", + "arraysize": "8", + "search_priority": "9", + "default_value": "NULL", + "type": "long" + }, + { + "name": "zoneID", + "db_type": "INTEGER", + "column_name": "zoneID", + "description": "Local zone index, found by dividing the sky into bands of declination 1/2 arcminute in height: zoneID = floor((90 + declination)/0.0083333).", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "10", + "arraysize": "4", + "search_priority": "10", + "default_value": "NULL", + "type": "int" + }, + { + "name": "tessID", + "db_type": "TINYINT", + "column_name": "tessID", + "description": "Tessellation identifier. Details in the TessellationType table.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "unsignedByte", + "size": "1", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "11", + "arraysize": "1", + "search_priority": "11", + "default_value": "NULL", + "type": "unsignedByte" + }, + { + "name": "projectionID", + "db_type": "SMALLINT", + "column_name": "projectionID", + "description": "Projection cell identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "12", + "arraysize": "2", + "search_priority": "12", + "default_value": "NULL", + "type": "short" + }, + { + "name": "skyCellID", + "db_type": "TINYINT", + "column_name": "skyCellID", + "description": "Skycell region identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "unsignedByte", + "size": "1", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "13", + "arraysize": "1", + "search_priority": "13", + "default_value": "NULL", + "type": "unsignedByte" + }, + { + "name": "randomID", + "db_type": "DOUBLE", + "column_name": "randomID", + "description": "Random value drawn from the interval between zero and one.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "14", + "arraysize": "8", + "search_priority": "14", + "default_value": "NULL", + "type": "double" + }, + { + "name": "batchID", + "db_type": "BIGINT", + "column_name": "batchID", + "description": "Internal database batch identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "long", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "15", + "arraysize": "8", + "search_priority": "15", + "default_value": "NULL", + "type": "long" + }, + { + "name": "dvoRegionID", + "db_type": "INTEGER", + "column_name": "dvoRegionID", + "description": "Internal DVO region identifier.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "16", + "arraysize": "4", + "search_priority": "16", + "default_value": "NULL", + "type": "int" + }, + { + "name": "processingVersion", + "db_type": "TINYINT", + "column_name": "processingVersion", + "description": "Data release version.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "unsignedByte", + "size": "1", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "17", + "arraysize": "1", + "search_priority": "17", + "default_value": "NULL", + "type": "unsignedByte" + }, + { + "name": "objInfoFlag", + "db_type": "INTEGER", + "column_name": "objInfoFlag", + "description": "Information flag bitmask indicating details of the photometry. Values listed in ObjectInfoFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "18", + "arraysize": "4", + "search_priority": "18", + "default_value": "NULL", + "type": "int" + }, + { + "name": "qualityFlag", + "db_type": "TINYINT", + "column_name": "qualityFlag", + "description": "Subset of objInfoFlag denoting whether this object is real or a likely false positive. Values listed in ObjectQualityFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "unsignedByte", + "size": "1", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "19", + "arraysize": "1", + "search_priority": "19", + "default_value": "NULL", + "type": "unsignedByte" + }, + { + "name": "raStack", + "db_type": "DOUBLE", + "column_name": "raStack", + "description": "Right ascension from stack detections, weighted mean value across filters, in equinox J2000. See StackObjectThin for stack epoch information.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "20", + "arraysize": "8", + "search_priority": "20", + "default_value": "-999", + "type": "double" + }, + { + "name": "decStack", + "db_type": "DOUBLE", + "column_name": "decStack", + "description": "Declination from stack detections, weighted mean value across filters, in equinox J2000. See StackObjectThin for stack epoch information.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "21", + "arraysize": "8", + "search_priority": "21", + "default_value": "-999", + "type": "double" + }, + { + "name": "raStackErr", + "db_type": "REAL", + "column_name": "raStackErr", + "description": "Right ascension standard deviation from stack detections.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "22", + "arraysize": "4", + "search_priority": "22", + "default_value": "-999", + "type": "float" + }, + { + "name": "decStackErr", + "db_type": "REAL", + "column_name": "decStackErr", + "description": "Declination standard deviation from stack detections.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "23", + "arraysize": "4", + "search_priority": "23", + "default_value": "-999", + "type": "float" + }, + { + "name": "raMean", + "db_type": "DOUBLE", + "column_name": "raMean", + "description": "Right ascension from single epoch detections (weighted mean) in equinox J2000 at the mean epoch given by epochMean.", + "unit": "deg", + "ucd": "pos.eq.ra;meta.main", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "24", + "arraysize": "8", + "search_priority": "24", + "default_value": "-999", + "type": "double" + }, + { + "name": "decMean", + "db_type": "DOUBLE", + "column_name": "decMean", + "description": "Declination from single epoch detections (weighted mean) in equinox J2000 at the mean epoch given by epochMean.", + "unit": "deg", + "ucd": "pos.eq.dec;meta.main", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "25", + "arraysize": "8", + "search_priority": "25", + "default_value": "-999", + "type": "double" + }, + { + "name": "raMeanErr", + "db_type": "DOUBLE", + "column_name": "raMeanErr", + "description": "Right ascension standard deviation from single epoch detections.", + "unit": "deg", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "26", + "arraysize": "4", + "search_priority": "26", + "default_value": "-999", + "type": "float" + }, + { + "name": "decMeanErr", + "db_type": "DOUBLE", + "column_name": "decMeanErr", + "description": "Declination standard deviation from single epoch detections.", + "unit": "deg", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "27", + "arraysize": "4", + "search_priority": "27", + "default_value": "-999", + "type": "float" + }, + { + "name": "epochMean", + "db_type": "DOUBLE", + "column_name": "epochMean", + "description": "Modified Julian Date of the mean epoch corresponding to raMean, decMean (equinox J2000).", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "28", + "arraysize": "8", + "search_priority": "28", + "default_value": "-999", + "type": "double" + }, + { + "name": "posMeanChisq", + "db_type": "DOUBLE", + "column_name": "posMeanChisq", + "description": "Reduced chi squared value of mean position.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "29", + "arraysize": "4", + "search_priority": "29", + "default_value": "-999", + "type": "float" + }, + { + "name": "cx", + "db_type": "DOUBLE", + "column_name": "cx", + "description": "Cartesian x on a unit sphere.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "30", + "arraysize": "8", + "search_priority": "30", + "default_value": "NULL", + "type": "double" + }, + { + "name": "cy", + "db_type": "DOUBLE", + "column_name": "cy", + "description": "Cartesian y on a unit sphere.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "31", + "arraysize": "8", + "search_priority": "31", + "default_value": "NULL", + "type": "double" + }, + { + "name": "cz", + "db_type": "DOUBLE", + "column_name": "cz", + "description": "Cartesian z on a unit sphere.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "32", + "arraysize": "8", + "search_priority": "32", + "default_value": "NULL", + "type": "double" + }, + { + "name": "lambda", + "db_type": "DOUBLE", + "column_name": "lambda", + "description": "Ecliptic longitude.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "33", + "arraysize": "8", + "search_priority": "33", + "default_value": "-999", + "type": "double" + }, + { + "name": "beta", + "db_type": "DOUBLE", + "column_name": "beta", + "description": "Ecliptic latitude.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "34", + "arraysize": "8", + "search_priority": "34", + "default_value": "-999", + "type": "double" + }, + { + "name": "l", + "db_type": "DOUBLE", + "column_name": "l", + "description": "Galactic longitude.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "35", + "arraysize": "8", + "search_priority": "35", + "default_value": "-999", + "type": "double" + }, + { + "name": "b", + "db_type": "DOUBLE", + "column_name": "b", + "description": "Galactic latitude.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "double", + "size": "8", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "36", + "arraysize": "8", + "search_priority": "36", + "default_value": "-999", + "type": "double" + }, + { + "name": "nStackObjectRows", + "db_type": "SMALLINT", + "column_name": "nStackObjectRows", + "description": "Number of independent StackObjectThin rows associated with this object.", + "unit": "NULL", + "ucd": "NULL", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "37", + "arraysize": "2", + "search_priority": "37", + "default_value": "-999", + "type": "short" + }, + { + "name": "nStackDetections", + "db_type": "SMALLINT", + "column_name": "nStackDetections", + "description": "Number of stack detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "caom:ps1.catalogEntity.nObs", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "38", + "arraysize": "2", + "search_priority": "38", + "default_value": "-999", + "type": "short" + }, + { + "name": "nDetections", + "db_type": "SMALLINT", + "column_name": "nDetections", + "description": "Number of single epoch detections in all filters.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "caom:ps1.catalogEntity.nObs", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "39", + "arraysize": "2", + "search_priority": "39", + "default_value": "-999", + "type": "short" + }, + { + "name": "ng", + "db_type": "SMALLINT", + "column_name": "ng", + "description": "Number of single epoch detections in g filter.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "40", + "arraysize": "2", + "search_priority": "40", + "default_value": "-999", + "type": "short" + }, + { + "name": "nr", + "db_type": "SMALLINT", + "column_name": "nr", + "description": "Number of single epoch detections in r filter.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "41", + "arraysize": "2", + "search_priority": "41", + "default_value": "-999", + "type": "short" + }, + { + "name": "ni", + "db_type": "SMALLINT", + "column_name": "ni", + "description": "Number of single epoch detections in i filter.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "42", + "arraysize": "2", + "search_priority": "42", + "default_value": "-999", + "type": "short" + }, + { + "name": "nz", + "db_type": "SMALLINT", + "column_name": "nz", + "description": "Number of single epoch detections in z filter.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "43", + "arraysize": "2", + "search_priority": "43", + "default_value": "-999", + "type": "short" + }, + { + "name": "ny", + "db_type": "SMALLINT", + "column_name": "ny", + "description": "Number of single epoch detections in y filter.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "44", + "arraysize": "2", + "search_priority": "44", + "default_value": "-999", + "type": "short" + }, + { + "name": "gQfPerfect", + "db_type": "REAL", + "column_name": "gQfPerfect", + "description": "Maximum PSF weighted fraction of pixels totally unmasked from g filter detections.", + "unit": "NULL", + "ucd": "stat.value", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "45", + "arraysize": "4", + "search_priority": "45", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanPSFMag", + "db_type": "REAL", + "column_name": "gMeanPSFMag", + "description": "Mean PSF magnitude from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "caom:ps1.catalogEntity.mag", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "46", + "arraysize": "4", + "search_priority": "46", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanPSFMagErr", + "db_type": "REAL", + "column_name": "gMeanPSFMagErr", + "description": "Error in mean PSF magnitude from g filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "47", + "arraysize": "4", + "search_priority": "47", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanPSFMagStd", + "db_type": "REAL", + "column_name": "gMeanPSFMagStd", + "description": "Standard deviation of PSF magnitudes from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "48", + "arraysize": "4", + "search_priority": "48", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanPSFMagNpt", + "db_type": "SMALLINT", + "column_name": "gMeanPSFMagNpt", + "description": "Number of measurements included in mean PSF magnitude from g filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "49", + "arraysize": "2", + "search_priority": "49", + "default_value": "-999", + "type": "short" + }, + { + "name": "gMeanPSFMagMin", + "db_type": "REAL", + "column_name": "gMeanPSFMagMin", + "description": "Minimum PSF magnitude from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.min", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "50", + "arraysize": "4", + "search_priority": "50", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanPSFMagMax", + "db_type": "REAL", + "column_name": "gMeanPSFMagMax", + "description": "Maximum PSF magnitude from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.max", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "51", + "arraysize": "4", + "search_priority": "51", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanKronMag", + "db_type": "REAL", + "column_name": "gMeanKronMag", + "description": "Mean Kron (1980) magnitude from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "52", + "arraysize": "4", + "search_priority": "52", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanKronMagErr", + "db_type": "REAL", + "column_name": "gMeanKronMagErr", + "description": "Error in mean Kron (1980) magnitude from g filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "53", + "arraysize": "4", + "search_priority": "53", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanKronMagStd", + "db_type": "REAL", + "column_name": "gMeanKronMagStd", + "description": "Standard deviation of Kron (1980) magnitudes from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "54", + "arraysize": "4", + "search_priority": "54", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanKronMagNpt", + "db_type": "SMALLINT", + "column_name": "gMeanKronMagNpt", + "description": "Number of measurements included in mean Kron (1980) magnitude from g filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "55", + "arraysize": "2", + "search_priority": "55", + "default_value": "-999", + "type": "short" + }, + { + "name": "gMeanApMag", + "db_type": "REAL", + "column_name": "gMeanApMag", + "description": "Mean aperture magnitude from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "56", + "arraysize": "4", + "search_priority": "56", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanApMagErr", + "db_type": "REAL", + "column_name": "gMeanApMagErr", + "description": "Error in mean aperture magnitude from g filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "57", + "arraysize": "4", + "search_priority": "57", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanApMagStd", + "db_type": "REAL", + "column_name": "gMeanApMagStd", + "description": "Standard deviation of aperture magnitudes from g filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "58", + "arraysize": "4", + "search_priority": "58", + "default_value": "-999", + "type": "float" + }, + { + "name": "gMeanApMagNpt", + "db_type": "SMALLINT", + "column_name": "gMeanApMagNpt", + "description": "Number of measurements included in mean aperture magnitude from g filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "59", + "arraysize": "2", + "search_priority": "59", + "default_value": "-999", + "type": "short" + }, + { + "name": "gFlags", + "db_type": "INTEGER", + "column_name": "gFlags", + "description": "Information flag bitmask for mean object from g filter detections. Values listed in ObjectFilterFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "60", + "arraysize": "4", + "search_priority": "60", + "default_value": "NULL", + "type": "int" + }, + { + "name": "rQfPerfect", + "db_type": "REAL", + "column_name": "rQfPerfect", + "description": "Maximum PSF weighted fraction of pixels totally unmasked from r filter detections.", + "unit": "NULL", + "ucd": "stat.value", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "61", + "arraysize": "4", + "search_priority": "61", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanPSFMag", + "db_type": "REAL", + "column_name": "rMeanPSFMag", + "description": "Mean PSF magnitude from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "caom:ps1.catalogEntity.mag", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "62", + "arraysize": "4", + "search_priority": "62", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanPSFMagErr", + "db_type": "REAL", + "column_name": "rMeanPSFMagErr", + "description": "Error in mean PSF magnitude from r filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "63", + "arraysize": "4", + "search_priority": "63", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanPSFMagStd", + "db_type": "REAL", + "column_name": "rMeanPSFMagStd", + "description": "Standard deviation of PSF magnitudes from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "64", + "arraysize": "4", + "search_priority": "64", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanPSFMagNpt", + "db_type": "SMALLINT", + "column_name": "rMeanPSFMagNpt", + "description": "Number of measurements included in mean PSF magnitude from r filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "65", + "arraysize": "2", + "search_priority": "65", + "default_value": "-999", + "type": "short" + }, + { + "name": "rMeanPSFMagMin", + "db_type": "REAL", + "column_name": "rMeanPSFMagMin", + "description": "Minimum PSF magnitude from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.min", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "66", + "arraysize": "4", + "search_priority": "66", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanPSFMagMax", + "db_type": "REAL", + "column_name": "rMeanPSFMagMax", + "description": "Maximum PSF magnitude from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.max", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "67", + "arraysize": "4", + "search_priority": "67", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanKronMag", + "db_type": "REAL", + "column_name": "rMeanKronMag", + "description": "Mean Kron (1980) magnitude from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "68", + "arraysize": "4", + "search_priority": "68", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanKronMagErr", + "db_type": "REAL", + "column_name": "rMeanKronMagErr", + "description": "Error in mean Kron (1980) magnitude from r filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "69", + "arraysize": "4", + "search_priority": "69", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanKronMagStd", + "db_type": "REAL", + "column_name": "rMeanKronMagStd", + "description": "Standard deviation of Kron (1980) magnitudes from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "70", + "arraysize": "4", + "search_priority": "70", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanKronMagNpt", + "db_type": "SMALLINT", + "column_name": "rMeanKronMagNpt", + "description": "Number of measurements included in mean Kron (1980) magnitude from r filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "71", + "arraysize": "2", + "search_priority": "71", + "default_value": "-999", + "type": "short" + }, + { + "name": "rMeanApMag", + "db_type": "REAL", + "column_name": "rMeanApMag", + "description": "Mean aperture magnitude from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "72", + "arraysize": "4", + "search_priority": "72", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanApMagErr", + "db_type": "REAL", + "column_name": "rMeanApMagErr", + "description": "Error in mean aperture magnitude from r filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "73", + "arraysize": "4", + "search_priority": "73", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanApMagStd", + "db_type": "REAL", + "column_name": "rMeanApMagStd", + "description": "Standard deviation of aperture magnitudes from r filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "74", + "arraysize": "4", + "search_priority": "74", + "default_value": "-999", + "type": "float" + }, + { + "name": "rMeanApMagNpt", + "db_type": "SMALLINT", + "column_name": "rMeanApMagNpt", + "description": "Number of measurements included in mean aperture magnitude from r filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "75", + "arraysize": "2", + "search_priority": "75", + "default_value": "-999", + "type": "short" + }, + { + "name": "rFlags", + "db_type": "INTEGER", + "column_name": "rFlags", + "description": "Information flag bitmask for mean object from r filter detections. Values listed in ObjectFilterFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "76", + "arraysize": "4", + "search_priority": "76", + "default_value": "NULL", + "type": "int" + }, + { + "name": "iQfPerfect", + "db_type": "REAL", + "column_name": "iQfPerfect", + "description": "Maximum PSF weighted fraction of pixels totally unmasked from i filter detections.", + "unit": "NULL", + "ucd": "stat.value", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "77", + "arraysize": "4", + "search_priority": "77", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanPSFMag", + "db_type": "REAL", + "column_name": "iMeanPSFMag", + "description": "Mean PSF magnitude from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "caom:ps1.catalogEntity.mag", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "78", + "arraysize": "4", + "search_priority": "78", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanPSFMagErr", + "db_type": "REAL", + "column_name": "iMeanPSFMagErr", + "description": "Error in mean PSF magnitude from i filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "79", + "arraysize": "4", + "search_priority": "79", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanPSFMagStd", + "db_type": "REAL", + "column_name": "iMeanPSFMagStd", + "description": "Standard deviation of PSF magnitudes from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "80", + "arraysize": "4", + "search_priority": "80", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanPSFMagNpt", + "db_type": "SMALLINT", + "column_name": "iMeanPSFMagNpt", + "description": "Number of measurements included in mean PSF magnitude from i filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "81", + "arraysize": "2", + "search_priority": "81", + "default_value": "-999", + "type": "short" + }, + { + "name": "iMeanPSFMagMin", + "db_type": "REAL", + "column_name": "iMeanPSFMagMin", + "description": "Minimum PSF magnitude from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.min", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "82", + "arraysize": "4", + "search_priority": "82", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanPSFMagMax", + "db_type": "REAL", + "column_name": "iMeanPSFMagMax", + "description": "Maximum PSF magnitude from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.max", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "83", + "arraysize": "4", + "search_priority": "83", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanKronMag", + "db_type": "REAL", + "column_name": "iMeanKronMag", + "description": "Mean Kron (1980) magnitude from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "84", + "arraysize": "4", + "search_priority": "84", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanKronMagErr", + "db_type": "REAL", + "column_name": "iMeanKronMagErr", + "description": "Error in mean Kron (1980) magnitude from i filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "85", + "arraysize": "4", + "search_priority": "85", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanKronMagStd", + "db_type": "REAL", + "column_name": "iMeanKronMagStd", + "description": "Standard deviation of Kron (1980) magnitudes from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "86", + "arraysize": "4", + "search_priority": "86", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanKronMagNpt", + "db_type": "SMALLINT", + "column_name": "iMeanKronMagNpt", + "description": "Number of measurements included in mean Kron (1980) magnitude from i filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "87", + "arraysize": "2", + "search_priority": "87", + "default_value": "-999", + "type": "short" + }, + { + "name": "iMeanApMag", + "db_type": "REAL", + "column_name": "iMeanApMag", + "description": "Mean aperture magnitude from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "88", + "arraysize": "4", + "search_priority": "88", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanApMagErr", + "db_type": "REAL", + "column_name": "iMeanApMagErr", + "description": "Error in mean aperture magnitude from i filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "89", + "arraysize": "4", + "search_priority": "89", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanApMagStd", + "db_type": "REAL", + "column_name": "iMeanApMagStd", + "description": "Standard deviation of aperture magnitudes from i filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "90", + "arraysize": "4", + "search_priority": "90", + "default_value": "-999", + "type": "float" + }, + { + "name": "iMeanApMagNpt", + "db_type": "SMALLINT", + "column_name": "iMeanApMagNpt", + "description": "Number of measurements included in mean aperture magnitude from i filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "91", + "arraysize": "2", + "search_priority": "91", + "default_value": "-999", + "type": "short" + }, + { + "name": "iFlags", + "db_type": "INTEGER", + "column_name": "iFlags", + "description": "Information flag bitmask for mean object from i filter detections. Values listed in ObjectFilterFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "92", + "arraysize": "4", + "search_priority": "92", + "default_value": "NULL", + "type": "int" + }, + { + "name": "zQfPerfect", + "db_type": "REAL", + "column_name": "zQfPerfect", + "description": "Maximum PSF weighted fraction of pixels totally unmasked from z filter detections.", + "unit": "NULL", + "ucd": "stat.value", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "93", + "arraysize": "4", + "search_priority": "93", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanPSFMag", + "db_type": "REAL", + "column_name": "zMeanPSFMag", + "description": "Mean PSF magnitude from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "caom:ps1.catalogEntity.mag", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "94", + "arraysize": "4", + "search_priority": "94", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanPSFMagErr", + "db_type": "REAL", + "column_name": "zMeanPSFMagErr", + "description": "Error in mean PSF magnitude from z filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "95", + "arraysize": "4", + "search_priority": "95", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanPSFMagStd", + "db_type": "REAL", + "column_name": "zMeanPSFMagStd", + "description": "Standard deviation of PSF magnitudes from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "96", + "arraysize": "4", + "search_priority": "96", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanPSFMagNpt", + "db_type": "SMALLINT", + "column_name": "zMeanPSFMagNpt", + "description": "Number of measurements included in mean PSF magnitude from z filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "97", + "arraysize": "2", + "search_priority": "97", + "default_value": "-999", + "type": "short" + }, + { + "name": "zMeanPSFMagMin", + "db_type": "REAL", + "column_name": "zMeanPSFMagMin", + "description": "Minimum PSF magnitude from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.min", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "98", + "arraysize": "4", + "search_priority": "98", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanPSFMagMax", + "db_type": "REAL", + "column_name": "zMeanPSFMagMax", + "description": "Maximum PSF magnitude from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.max", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "99", + "arraysize": "4", + "search_priority": "99", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanKronMag", + "db_type": "REAL", + "column_name": "zMeanKronMag", + "description": "Mean Kron (1980) magnitude from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "100", + "arraysize": "4", + "search_priority": "100", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanKronMagErr", + "db_type": "REAL", + "column_name": "zMeanKronMagErr", + "description": "Error in mean Kron (1980) magnitude from z filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "101", + "arraysize": "4", + "search_priority": "101", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanKronMagStd", + "db_type": "REAL", + "column_name": "zMeanKronMagStd", + "description": "Standard deviation of Kron (1980) magnitudes from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "102", + "arraysize": "4", + "search_priority": "102", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanKronMagNpt", + "db_type": "SMALLINT", + "column_name": "zMeanKronMagNpt", + "description": "Number of measurements included in mean Kron (1980) magnitude from z filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "103", + "arraysize": "2", + "search_priority": "103", + "default_value": "-999", + "type": "short" + }, + { + "name": "zMeanApMag", + "db_type": "REAL", + "column_name": "zMeanApMag", + "description": "Mean aperture magnitude from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "104", + "arraysize": "4", + "search_priority": "104", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanApMagErr", + "db_type": "REAL", + "column_name": "zMeanApMagErr", + "description": "Error in mean aperture magnitude from z filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "105", + "arraysize": "4", + "search_priority": "105", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanApMagStd", + "db_type": "REAL", + "column_name": "zMeanApMagStd", + "description": "Standard deviation of aperture magnitudes from z filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "106", + "arraysize": "4", + "search_priority": "106", + "default_value": "-999", + "type": "float" + }, + { + "name": "zMeanApMagNpt", + "db_type": "SMALLINT", + "column_name": "zMeanApMagNpt", + "description": "Number of measurements included in mean aperture magnitude from z filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "107", + "arraysize": "2", + "search_priority": "107", + "default_value": "-999", + "type": "short" + }, + { + "name": "zFlags", + "db_type": "INTEGER", + "column_name": "zFlags", + "description": "Information flag bitmask for mean object from z filter detections. Values listed in ObjectFilterFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "108", + "arraysize": "4", + "search_priority": "108", + "default_value": "NULL", + "type": "int" + }, + { + "name": "yQfPerfect", + "db_type": "REAL", + "column_name": "yQfPerfect", + "description": "Maximum PSF weighted fraction of pixels totally unmasked from y filter detections.", + "unit": "NULL", + "ucd": "stat.value", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "109", + "arraysize": "4", + "search_priority": "109", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanPSFMag", + "db_type": "REAL", + "column_name": "yMeanPSFMag", + "description": "Mean PSF magnitude from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;em:ir;meta.main", + "utype": "caom:ps1.catalogEntity.mag", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "110", + "arraysize": "4", + "search_priority": "110", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanPSFMagErr", + "db_type": "REAL", + "column_name": "yMeanPSFMagErr", + "description": "Error in mean PSF magnitude from y filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "111", + "arraysize": "4", + "search_priority": "111", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanPSFMagStd", + "db_type": "REAL", + "column_name": "yMeanPSFMagStd", + "description": "Standard deviation of PSF magnitudes from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "112", + "arraysize": "4", + "search_priority": "112", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanPSFMagNpt", + "db_type": "SMALLINT", + "column_name": "yMeanPSFMagNpt", + "description": "Number of measurements included in mean PSF magnitude from y filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "113", + "arraysize": "2", + "search_priority": "113", + "default_value": "-999", + "type": "short" + }, + { + "name": "yMeanPSFMagMin", + "db_type": "REAL", + "column_name": "yMeanPSFMagMin", + "description": "Minimum PSF magnitude from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.min", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "114", + "arraysize": "4", + "search_priority": "114", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanPSFMagMax", + "db_type": "REAL", + "column_name": "yMeanPSFMagMax", + "description": "Maximum PSF magnitude from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.max", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "115", + "arraysize": "4", + "search_priority": "115", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanKronMag", + "db_type": "REAL", + "column_name": "yMeanKronMag", + "description": "Mean Kron (1980) magnitude from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;em:ir;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "116", + "arraysize": "4", + "search_priority": "116", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanKronMagErr", + "db_type": "REAL", + "column_name": "yMeanKronMagErr", + "description": "Error in mean Kron (1980) magnitude from y filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "117", + "arraysize": "4", + "search_priority": "117", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanKronMagStd", + "db_type": "REAL", + "column_name": "yMeanKronMagStd", + "description": "Standard deviation of Kron (1980) magnitudes from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "118", + "arraysize": "4", + "search_priority": "118", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanKronMagNpt", + "db_type": "SMALLINT", + "column_name": "yMeanKronMagNpt", + "description": "Number of measurements included in mean Kron (1980) magnitude from y filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "119", + "arraysize": "2", + "search_priority": "119", + "default_value": "-999", + "type": "short" + }, + { + "name": "yMeanApMag", + "db_type": "REAL", + "column_name": "yMeanApMag", + "description": "Mean aperture magnitude from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;em:ir;meta.main", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "120", + "arraysize": "4", + "search_priority": "120", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanApMagErr", + "db_type": "REAL", + "column_name": "yMeanApMagErr", + "description": "Error in mean aperture magnitude from y filter detections.", + "unit": "mag", + "ucd": "stat.error", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "121", + "arraysize": "4", + "search_priority": "121", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanApMagStd", + "db_type": "REAL", + "column_name": "yMeanApMagStd", + "description": "Standard deviation of aperture magnitudes from y filter detections.", + "unit": "mag", + "ucd": "phot.mag;em.opt;stat.stdev", + "utype": "NULL", + "datatype": "float", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "122", + "arraysize": "4", + "search_priority": "122", + "default_value": "-999", + "type": "float" + }, + { + "name": "yMeanApMagNpt", + "db_type": "SMALLINT", + "column_name": "yMeanApMagNpt", + "description": "Number of measurements included in mean aperture magnitude from y filter detections.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "short", + "size": "2", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "123", + "arraysize": "2", + "search_priority": "123", + "default_value": "-999", + "type": "short" + }, + { + "name": "yFlags", + "db_type": "INTEGER", + "column_name": "yFlags", + "description": "Information flag bitmask for mean object from y filter detections. Values listed in ObjectFilterFlags.", + "unit": "NULL", + "ucd": "meta.code", + "utype": "NULL", + "datatype": "int", + "size": "4", + "principal": "NULL", + "indexed": "NULL", + "std": "NULL", + "column_index": "124", + "arraysize": "4", + "search_priority": "124", + "default_value": "NULL", + "type": "int" + } +] \ No newline at end of file diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 3b47b23e92..282d9f0648 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -47,6 +47,7 @@ 'Mast.HscMatches.Db.v2': 'matchid.json', 'Mast.HscSpectra.Db.All': 'spectra.json', 'panstarrs': 'panstarrs.json', + 'panstarrs_columns': 'panstarrs_columns.json', 'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip', 'tess_sector': 'tess_sector.json', 'z_cutout_fit': 'astrocut_189.49206_62.20615_100x100px_f.zip', @@ -129,11 +130,13 @@ def service_mockreturn(self, method="POST", url=None, data=None, timeout=10, use return MockResponse(content) -def request_mockreturn(url, params=None): +def request_mockreturn(url, params={}): if 'column_list' in url: filename = data_path(DATA_FILES['mission_columns']) elif 'Mast.Name.Lookup' in params: filename = data_path(DATA_FILES["Mast.Name.Lookup"]) + elif 'panstarrs' in url: + filename = data_path(DATA_FILES['panstarrs_columns']) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -298,7 +301,6 @@ def test_mast_query(patch_post): def test_resolve_object(patch_post): m103_loc = mast.Mast.resolve_object("M103") - print(m103_loc) assert round(m103_loc.separation(SkyCoord("23.34086 60.658", unit='deg')).value, 10) == 0 diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 001bfc2fe6..89a5cd6f00 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -963,6 +963,25 @@ def test_catalogs_query_criteria_invalid_keyword(self): Catalogs.query_criteria(catalog='ctl', objectType="STAR") assert 'objType' in str(err_with_alt.value) + # region query with invalid keyword + with pytest.raises(InvalidQueryError) as err_region: + Catalogs.query_region('322.49324 12.16683', + radius=0.001*u.deg, + catalog='HSC', + invalid=2) + assert "Filter 'invalid' does not exist for catalog HSC." in str(err_region.value) + + # panstarrs criteria query with invalid keyword + with pytest.raises(InvalidQueryError) as err_ps_criteria: + Catalogs.query_criteria(coordinates="158.47924 -7.30962", + catalog="PANSTARRS", + table="mean", + data_release="dr2", + columns=["objName", "distance"], + sort_by=[("asc", "distance")], + obj_name='invalid') + assert 'objName' in str(err_ps_criteria.value) + def test_catalogs_query_hsc_matchid_async(self): catalogData = Catalogs.query_object("M10", radius=.001, From ed3bdffeeb1971d7483c9ed43733ac9b9cd99465 Mon Sep 17 00:00:00 2001 From: Sam Bianco Date: Wed, 6 Nov 2024 17:10:20 -0500 Subject: [PATCH 5/5] Get cloud URIs for HLSPs test case, changelog --- CHANGES.rst | 4 +++- astroquery/mast/tests/test_mast_remote.py | 2 +- astroquery/mast/utils.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 897d7f82d6..8322e42cef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -226,7 +226,9 @@ mast - Present users with an error when nonexistent query criteria are used in ``mast.MastMissions`` query functions. [#3126] - Present users with an error when nonexistent query criteria are used in ``mast.Catalogs.query_region`` and - ``mast.Catalogs.query_object`` [#3126] + ``mast.Catalogs.query_object``. [#3126] + +- Handle HLSP data products in ``Observations.get_cloud_uris``. [#3126] mpc ^^^ diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 89a5cd6f00..08f960c647 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -605,7 +605,7 @@ def test_get_cloud_uri(self, test_data_uri, expected_cloud_uri): assert len(uri) > 0, f'Product for dataURI {test_data_uri} was not found in the cloud.' assert uri == expected_cloud_uri, f'Cloud URI does not match expected. ({uri} != {expected_cloud_uri})' - @pytest.mark.parametrize("test_obs_id", ["25568122", "31411"]) + @pytest.mark.parametrize("test_obs_id", ["25568122", "31411", "107604081"]) def test_get_cloud_uris(self, test_obs_id): pytest.importorskip("boto3") diff --git a/astroquery/mast/utils.py b/astroquery/mast/utils.py index 9f7a25d731..56bfa5810a 100644 --- a/astroquery/mast/utils.py +++ b/astroquery/mast/utils.py @@ -194,6 +194,8 @@ def mast_relative_path(mast_uri): path = path.lstrip("/mast/") elif '/ps1/' in path: path = path.replace("/ps1/", "panstarrs/ps1/public/") + elif 'hlsp' in path: + path = path.replace("/hlsp_local/public/", "mast/") else: path = path.lstrip("/") result.append(path)